Replaced FieldStoragePathCapture with cleaner DataclassFieldLookup in dataclassio

.idea/dictionaries/ericf.xml

@@ -1139,6 +1139,7 @@
       <w>janktastic</w>
       <w>janky</w>
       <w>jascha</w>
+      <w>jdict</w>
       <w>jenkinsfile</w>
       <w>jexport</w>
       <w>jisx</w>
@@ -1623,6 +1624,7 @@
       <w>pathcapture</w>
       <w>pathlib</w>
       <w>pathnames</w>
+      <w>pathparts</w>
       <w>pathstonames</w>
       <w>pathtmp</w>
       <w>patsubst</w>

ballisticacore-cmake/.idea/dictionaries/ericf.xml

@@ -540,6 +540,7 @@
       <w>janky</w>
       <w>jaxis</w>
       <w>jcjwf</w>
+      <w>jdict</w>
       <w>jmessage</w>
       <w>jnames</w>
       <w>keepalives</w>
@@ -771,6 +772,7 @@
       <w>parameteriv</w>
       <w>passcode</w>
       <w>pathcapture</w>
+      <w>pathparts</w>
       <w>pausable</w>
       <w>pcommands</w>
       <w>pdataclass</w>

docs/ba_module.md

@@ -1,5 +1,5 @@
 <!-- THIS FILE IS AUTO GENERATED; DO NOT EDIT BY HAND -->
-<h4><em>last updated on 2021-09-29 for Ballistica version 1.6.5 build 20393</em></h4>
+<h4><em>last updated on 2021-10-01 for Ballistica version 1.6.5 build 20393</em></h4>
 <p>This page documents the Python classes and functions in the 'ba' module,
  which are the ones most relevant to modding in Ballistica. If you come across something you feel should be included here or could be better explained, please <a href="mailto:support@froemling.net">let me know</a>. Happy modding!</p>
 <hr>

tests/test_efro/test_dataclassio.py

@@ -16,7 +16,7 @@ import pytest
 from efro.util import utc_now
 from efro.dataclassio import (dataclass_validate, dataclass_from_dict,
                               dataclass_to_dict, ioprepped, IOAttrs, Codec,
-                              FieldStoragePathCapture)
+                              DataclassFieldLookup)
 
 if TYPE_CHECKING:
     pass
@@ -697,20 +697,6 @@ class _SPTestClass2:
                      IOAttrs('s')] = field(default_factory=_SPTestClass1)
 
 
-def test_field_storage_path_capture() -> None:
-    """Test FieldStoragePathCapture functionality."""
-
-    obj = _SPTestClass2()
-
-    namecap = FieldStoragePathCapture(obj)
-    assert namecap.subc.barf == 'subc.barf'
-    assert namecap.subc2.barf == 's.barf'
-    assert namecap.subc2.barf2 == 's.b'
-
-    with pytest.raises(AttributeError):
-        assert namecap.nonexistent.barf == 's.barf'
-
-
 def test_datetime_limits() -> None:
     """Test limiting datetime values in various ways."""
     from efro.util import utc_today, utc_this_hour
@@ -752,6 +738,48 @@ def test_datetime_limits() -> None:
         dataclass_from_dict(_TestClass2, out)
 
 
+def test_field_paths() -> None:
+    """Test type-safe field path evaluations."""
+
+    # Define a few nested dataclass types, some of which
+    # have storage names differing from their field names.
+    @ioprepped
+    @dataclass
+    class _TestClass:
+
+        @dataclass
+        class _TestSubClass:
+            val1: int = 0
+            val2: Annotated[int, IOAttrs('v2')] = 0
+
+        sub1: _TestSubClass = field(default_factory=_TestSubClass)
+        sub2: Annotated[_TestSubClass,
+                        IOAttrs('s2')] = field(default_factory=_TestSubClass)
+
+    # Now let's lookup various storage paths.
+    lookup = DataclassFieldLookup(_TestClass)
+
+    # Make sure lookups are returning correct storage paths.
+    assert lookup.path(lambda obj: obj.sub1) == 'sub1'
+    assert lookup.path(lambda obj: obj.sub1.val1) == 'sub1.val1'
+    assert lookup.path(lambda obj: obj.sub1.val2) == 'sub1.v2'
+    assert lookup.path(lambda obj: obj.sub2.val1) == 's2.val1'
+    assert lookup.path(lambda obj: obj.sub2.val2) == 's2.v2'
+
+    # Attempting to return fields that aren't there should fail
+    # in both type-checking and runtime.
+    with pytest.raises(AttributeError):
+        lookup.path(lambda obj: obj.sub1.val3)  # type: ignore
+
+    # Returning non-field objects will fail at runtime
+    # even if type-checking evaluates them as valid values.
+    with pytest.raises(TypeError):
+        lookup.path(lambda obj: 1)
+
+    with pytest.raises(TypeError):
+        lookup.path(lambda obj: obj.sub1.val1.real)
+
+
 def test_nested() -> None:
     """Test nesting dataclasses."""
 

tools/bacommon/assets.py

@@ -4,10 +4,14 @@
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Dict, Optional, List
 from enum import Enum
 
-from efro import entity
+from typing_extensions import Annotated
+
+from efro.dataclassio import ioprepped, IOAttrs
+# from efro import entity
 
 if TYPE_CHECKING:
     pass
@@ -33,26 +37,27 @@ class AssetType(Enum):
     COLLISION_MESH = 'collision_mesh'
 
 
-class AssetPackageFlavorManifestValue(entity.CompoundValue):
+@ioprepped
+@dataclass
+class AssetPackageFlavorManifest:
     """A manifest of asset info for a specific flavor of an asset package."""
-    assetfiles = entity.DictField('assetfiles', str, entity.StringValue())
+    assetfiles: Annotated[Dict[str, str],
+                          IOAttrs('assetfiles')] = field(default_factory=dict)
 
 
-class AssetPackageFlavorManifest(entity.EntityMixin,
+@ioprepped
-                                 AssetPackageFlavorManifestValue):
+@dataclass
-    """A self contained AssetPackageFlavorManifestValue."""
+class AssetPackageBuildState:
-
-
-class AssetPackageBuildState(entity.Entity):
     """Contains info about an in-progress asset cloud build."""
 
     # Asset names still being built.
-    in_progress_builds = entity.ListField('b', entity.StringValue())
+    in_progress_builds: Annotated[List[str],
+                                  IOAttrs('b')] = field(default_factory=list)
 
     # The initial number of assets needing to be built.
-    initial_build_count = entity.Field('c', entity.IntValue())
+    initial_build_count: Annotated[int, IOAttrs('c')] = 0
 
     # Build error string. If this is present, it should be presented
     # to the user and they should required to explicitly restart the build
     # in some way if desired.
-    error = entity.Field('e', entity.OptionalStringValue())
+    error: Annotated[Optional[str], IOAttrs('e')] = None

tools/efro/dataclassio/__init__.py

@@ -16,14 +16,14 @@ from efro.dataclassio._outputter import _Outputter
 from efro.dataclassio._inputter import _Inputter
 from efro.dataclassio._base import Codec, IOAttrs
 from efro.dataclassio._prep import ioprep, ioprepped, is_ioprepped_dataclass
-from efro.dataclassio._pathcapture import FieldStoragePathCapture
+from efro.dataclassio._pathcapture import DataclassFieldLookup
 
 if TYPE_CHECKING:
     from typing import Any, Dict, Type, Tuple, Optional, List, Set
 
 __all__ = [
     'Codec', 'IOAttrs', 'ioprep', 'ioprepped', 'is_ioprepped_dataclass',
-    'FieldStoragePathCapture', 'dataclass_to_dict', 'dataclass_to_json',
+    'DataclassFieldLookup', 'dataclass_to_dict', 'dataclass_to_json',
     'dataclass_from_dict', 'dataclass_from_json', 'dataclass_validate'
 ]
 
@@ -57,18 +57,20 @@ def dataclass_to_dict(obj: Any,
     return out
 
 
-def dataclass_to_json(obj: Any, coerce_to_float: bool = True) -> str:
+def dataclass_to_json(obj: Any,
+                      coerce_to_float: bool = True,
+                      pretty: bool = False) -> str:
     """Utility function; return a json string from a dataclass instance.
 
     Basically json.dumps(dataclass_to_dict(...)).
     """
     import json
-    return json.dumps(
+    jdict = dataclass_to_dict(obj=obj,
-        dataclass_to_dict(obj=obj,
+                              coerce_to_float=coerce_to_float,
-                          coerce_to_float=coerce_to_float,
+                              codec=Codec.JSON)
-                          codec=Codec.JSON),
+    if pretty:
-        separators=(',', ':'),
+        return json.dumps(jdict, indent=2, sort_keys=True)
-    )
+    return json.dumps(jdict, separators=(',', ':'))
 
 
 def dataclass_from_dict(cls: Type[T],

tools/efro/dataclassio/_pathcapture.py

@@ -5,39 +5,35 @@
 from __future__ import annotations
 
 import dataclasses
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, TypeVar, Generic
 
 from efro.dataclassio._base import _parse_annotated, _get_origin
 from efro.dataclassio._prep import PrepSession
 
 if TYPE_CHECKING:
-    from typing import Any, Dict, Type, Tuple, Optional, List, Set
+    from typing import Any, Dict, Type, Tuple, Optional, List, Set, Callable
+
+T = TypeVar('T')
 
 
-class FieldStoragePathCapture:
+class _PathCapture:
-    """Utility for obtaining dataclass storage paths in a type safe way.
+    """Utility for obtaining dataclass storage paths in a type safe way."""
 
-    Given dataclass instance foo, FieldStoragePathCapture(foo).bar.eep
+    def __init__(self, obj: Any, pathparts: List[str] = None):
-    will return 'bar.eep' (or something like 'b.e' if storagenames are
+        self._is_dataclass = dataclasses.is_dataclass(obj)
-    overridden). This can be combined with type-checking tricks that
+        if pathparts is None:
-    return foo in the type-checker's eyes while returning
+            pathparts = []
-    FieldStoragePathCapture(foo) at runtime in order to grant a measure
-    of type safety to specifying field paths for things such as db
-    queries. Be aware, however, that the type-checker will incorrectly
-    think these lookups are returning actual attr values when they
-    are actually returning strings.
-    """
-
-    def __init__(self, obj: Any, path: List[str] = None):
-        if path is None:
-            path = []
-        if not dataclasses.is_dataclass(obj):
-            raise TypeError(f'Expected a dataclass type/instance;'
-                            f' got {type(obj)}.')
         self._cls = obj if isinstance(obj, type) else type(obj)
-        self._path = path
+        self._pathparts = pathparts
+
+    def __getattr__(self, name: str) -> _PathCapture:
+
+        # We only allow diving into sub-objects if we are a dataclass.
+        if not self._is_dataclass:
+            raise TypeError(
+                f"Field path cannot include attribute '{name}' "
+                f'under parent {self._cls}; parent types must be dataclasses.')
 
-    def __getattr__(self, name: str) -> Any:
         prep = PrepSession(explicit=False).prep_dataclass(self._cls,
                                                           recursion_level=0)
         try:
@@ -48,8 +44,43 @@ class FieldStoragePathCapture:
         storagename = (name if (ioattrs is None or ioattrs.storagename is None)
                        else ioattrs.storagename)
         origin = _get_origin(anntype)
-        path = self._path + [storagename]
+        return _PathCapture(origin, pathparts=self._pathparts + [storagename])
 
-        if dataclasses.is_dataclass(origin):
+    @property
-            return FieldStoragePathCapture(origin, path=path)
+    def path(self) -> str:
-        return '.'.join(path)
+        """The final output path."""
+        return '.'.join(self._pathparts)
+
+
+class DataclassFieldLookup(Generic[T]):
+    """Get info about nested dataclass fields in type-safe way."""
+
+    def __init__(self, cls: T) -> None:
+        self.cls = cls
+
+    def path(self, callback: Callable[[T], Any]) -> str:
+        """Look up a path on child dataclass fields.
+
+        example:
+          DataclassFieldLookup(MyType).path(lambda obj: obj.foo.bar)
+
+        The above example will return the string 'foo.bar' or something
+        like 'f.b' if the dataclasses have custom storage names set.
+        It will also be static-type-checked, triggering an error if
+        MyType.foo.bar is not a valid path. Note, however, that the
+        callback technically allows any return value but only nested
+        dataclasses and their fields will succeed.
+        """
+
+        # We tell the type system that we are returning an instance
+        # of our class, which allows it to perform type checking on
+        # member lookups. In reality, however, we are providing a
+        # special object which captures path lookups so we can build
+        # a string from them.
+        if not TYPE_CHECKING:
+            out = callback(_PathCapture(self.cls))
+            if not isinstance(out, _PathCapture):
+                raise TypeError(f'Expected a valid path under'
+                                f' the provided object; got a {type(out)}.')
+            return out.path
+        return ''