Added tuple and datetime support to dataclassio

.efrocachemap

@@ -3940,18 +3940,18 @@
   "build/prefab/full/linux_x86_64/release/ballisticacore": "https://files.ballistica.net/cache/ba1/8b/8b/b8f8a75b3ded113231265f61da9d",
   "build/prefab/full/linux_x86_64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/14/4e/bd10863753f44c7612ef697c4693",
   "build/prefab/full/linux_x86_64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/34/a0/883c84cb130780bb8bc8a2185604",
-  "build/prefab/full/mac_arm64/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/d8/73/ef08a58076c8fc37af28bf097628",
+  "build/prefab/full/mac_arm64/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/e0/ee/e6b94bf4149530e412cfe27e5cb4",
   "build/prefab/full/mac_arm64/release/ballisticacore": "https://files.ballistica.net/cache/ba1/97/62/0944cd3ab34d681cd7c9bfaa0b11",
-  "build/prefab/full/mac_arm64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/9c/c0/ab01fbc7544f451db865b93e5430",
+  "build/prefab/full/mac_arm64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/dd/a5/c73d18aba833987ff3e713bbf981",
   "build/prefab/full/mac_arm64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/27/e1/de75aac52e10bebae81fc12aa030",
-  "build/prefab/full/mac_x86_64/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/d0/53/f358c93ad50992c3aee613034e38",
+  "build/prefab/full/mac_x86_64/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/a6/81/ee957a5bfd1be45c0e865f8a27ba",
   "build/prefab/full/mac_x86_64/release/ballisticacore": "https://files.ballistica.net/cache/ba1/03/29/abffbfc56fd981915253f1d4ed96",
-  "build/prefab/full/mac_x86_64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/a4/83/bf81025600a3a440d359057f0e05",
+  "build/prefab/full/mac_x86_64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/04/35/104446e5f91b9fe35fa413be02ca",
   "build/prefab/full/mac_x86_64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/ed/c0/68b44a693639308981b5214f02c1",
-  "build/prefab/full/windows_x86/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/5b/f8/e1934b7fd16e75bdf9dadccef22b",
+  "build/prefab/full/windows_x86/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/b5/c4/5935bc59cc237c42e8ac9be47ce5",
-  "build/prefab/full/windows_x86/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/1d/ad/193fc3816804c48954a6b2901b49",
+  "build/prefab/full/windows_x86/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/35/3d/f04ebd4a7d066088595b8ed4bbc5",
-  "build/prefab/full/windows_x86_server/debug/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/89/c5/0fcd0aafe25cb054f165b3ede0e5",
+  "build/prefab/full/windows_x86_server/debug/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/72/eb/25cf435771601aeec273a92ffb50",
-  "build/prefab/full/windows_x86_server/release/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/ae/2c/bfa29fcf8e83db1cc5bb32da1178",
+  "build/prefab/full/windows_x86_server/release/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/45/fb/166e932a6235613935ccf2e51d00",
   "build/prefab/lib/linux_arm64/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/f4/59/16c01f646c16bb480a197aa9e6e3",
   "build/prefab/lib/linux_arm64/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/81/73/a321fe4aa721d07f2a44257967cf",
   "build/prefab/lib/linux_arm64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/99/99/2adf5a22923eed3f8b5667ee8220",
@@ -3960,12 +3960,12 @@
   "build/prefab/lib/linux_x86_64/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/3d/3a/c747993afbf4c1ed1c5e8b0e5d5a",
   "build/prefab/lib/linux_x86_64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/46/21/31fe300afbf7d9da766c04064919",
   "build/prefab/lib/linux_x86_64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/f8/bd/2b05dbfd98cd55cedf924b639765",
-  "build/prefab/lib/mac_arm64/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/a9/1d/3896630323d201928a9a94d1132b",
+  "build/prefab/lib/mac_arm64/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/5b/6d/877a5a015eb3d705795a2db7fa74",
-  "build/prefab/lib/mac_arm64/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/39/eb/512862ff8b6e5aca44d4ef9b2b00",
+  "build/prefab/lib/mac_arm64/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/58/a4/3d2a1e4a47c51e883ef29b5e280d",
-  "build/prefab/lib/mac_arm64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/7c/db/2644a5447ec880608cc719211a03",
+  "build/prefab/lib/mac_arm64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/a7/ca/91cfd7dc2ccc8c996daf9aaeb9fd",
-  "build/prefab/lib/mac_arm64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/7a/d6/fe10f5e34d8c86a47ddbea041297",
+  "build/prefab/lib/mac_arm64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/93/cb/de559dd0ea55e12a228ea1cd8974",
-  "build/prefab/lib/mac_x86_64/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/f1/f8/0529ec1e41b71f9e595eff5d9655",
+  "build/prefab/lib/mac_x86_64/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/c9/e1/132123edad385ac0a977337f044c",
-  "build/prefab/lib/mac_x86_64/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/b8/e5/4acf3b0f3031a22ef02a8ec32a3d",
+  "build/prefab/lib/mac_x86_64/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/f2/e3/a82f88651f2ffef8c52a20da42a6",
-  "build/prefab/lib/mac_x86_64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/a4/c5/1ec7762968c5ab8a95d4b2203310",
+  "build/prefab/lib/mac_x86_64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/d4/9c/95e358c6e929bcb046e40fe5ad2b",
-  "build/prefab/lib/mac_x86_64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/84/23/0a99eea99bcf009a1801eb6c4755"
+  "build/prefab/lib/mac_x86_64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/8e/2d/81ce3feefa50c283cd7a8398ac72"
 }

.idea/dictionaries/ericf.xml

@@ -345,6 +345,7 @@
       <w>childnode</w>
       <w>childtype</w>
       <w>childtypes</w>
+      <w>childval</w>
       <w>chinesetraditional</w>
       <w>chipfork</w>
       <w>chosenone</w>
@@ -491,6 +492,7 @@
       <w>dataval</w>
       <w>datetimemodule</w>
       <w>datetimes</w>
+      <w>datetimeval</w>
       <w>daynum</w>
       <w>dayoffset</w>
       <w>dbapi</w>
@@ -2283,6 +2285,7 @@
       <w>tself</w>
       <w>tspc</w>
       <w>tstr</w>
+      <w>tupleval</w>
       <w>turtledemo</w>
       <w>tval</w>
       <w>tvalue</w>

Makefile

@@ -661,7 +661,7 @@ test-assetmanager:
       tests/test_ba/test_assetmanager.py::test_assetmanager
 
 # Individual test with extra output enabled.
-test-dataclasses:
+test-dataclassio:
 	@tools/pcommand pytest -o log_cli=true -o log_cli_level=debug -s -vv \
       tests/test_efro/test_dataclassio.py
 

ballisticacore-cmake/.idea/dictionaries/ericf.xml

@@ -164,6 +164,7 @@
       <w>childanntypes</w>
       <w>childtype</w>
       <w>childtypes</w>
+      <w>childval</w>
       <w>chrono</w>
       <w>chunksize</w>
       <w>cjief</w>
@@ -220,6 +221,7 @@
       <w>datas</w>
       <w>datav</w>
       <w>datavec</w>
+      <w>datetimeval</w>
       <w>dbgstr</w>
       <w>dbias</w>
       <w>dcioexattrs</w>
@@ -995,6 +997,7 @@
       <w>trimeshes</w>
       <w>trynum</w>
       <w>tself</w>
+      <w>tupleval</w>
       <w>tval</w>
       <w>tvos</w>
       <w>tweakage</w>

docs/ba_module.md

@@ -1,5 +1,5 @@
 <!-- THIS FILE IS AUTO GENERATED; DO NOT EDIT BY HAND -->
-<h4><em>last updated on 2021-04-30 for Ballistica version 1.6.0 build 20353</em></h4>
+<h4><em>last updated on 2021-05-03 for Ballistica version 1.6.0 build 20355</em></h4>
 <p>This page documents the Python classes and functions in the 'ba' module,
  which are the ones most relevant to modding in Ballistica. If you come across something you feel should be included here or could be better explained, please <a href="mailto:support@froemling.net">let me know</a>. Happy modding!</p>
 <hr>

tests/test_efro/test_dataclassio.py

@@ -5,16 +5,18 @@
 from __future__ import annotations
 
 from enum import Enum
+import datetime
 from dataclasses import field, dataclass
 from typing import TYPE_CHECKING
 
 import pytest
 
+from efro.util import utc_now
 from efro.dataclassio import (dataclass_validate, dataclass_from_dict,
                               dataclass_to_dict, prepped)
 
 if TYPE_CHECKING:
-    from typing import Optional, List, Set, Any, Dict, Sequence, Union
+    from typing import Optional, List, Set, Any, Dict, Sequence, Union, Tuple
 
 
 class _EnumTest(Enum):
@@ -75,6 +77,8 @@ def test_assign() -> None:
         ssval: Set[str] = field(default_factory=set)
         anyval: Any = 1
         dictval: Dict[int, str] = field(default_factory=dict)
+        tupleval: Tuple[int, str, bool] = (1, 'foo', False)
+        datetimeval: Optional[datetime.datetime] = None
 
     class _TestClass2:
         pass
@@ -89,14 +93,20 @@ def test_assign() -> None:
     with pytest.raises(TypeError):
         dataclass_from_dict(_TestClass, None)  # type: ignore
 
+    now = utc_now()
+
     # A dict containing *ALL* values should match what we
     # get when creating a dataclass and then converting back
     # to a dict.
     dict1 = {
-        'ival': 1,
+        'ival':
-        'sval': 'foo',
+            1,
-        'bval': True,
+        'sval':
-        'fval': 2.0,
+            'foo',
+        'bval':
+            True,
+        'fval':
+            2.0,
         'nval': {
             'ival': 1,
             'sval': 'bar',
@@ -104,12 +114,18 @@ def test_assign() -> None:
                 '1': 'foof'
             },
         },
-        'enval': 'test1',
+        'enval':
-        'oival': 1,
+            'test1',
-        'osval': 'foo',
+        'oival':
-        'obval': True,
+            1,
-        'ofval': 1.0,
+        'osval':
-        'oenval': 'test2',
+            'foo',
+        'obval':
+            True,
+        'ofval':
+            1.0,
+        'oenval':
+            'test2',
         'lsval': ['foo'],
         'lival': [10],
         'lbval': [False],
@@ -127,7 +143,12 @@ def test_assign() -> None:
         },
         'dictval': {
             '1': 'foo'
-        }
+        },
+        'tupleval': [2, 'foof', True],
+        'datetimeval': [
+            now.year, now.month, now.day, now.hour, now.minute, now.second,
+            now.microsecond
+        ],
     }
     dc1 = dataclass_from_dict(_TestClass, dict1)
     assert dataclass_to_dict(dc1) == dict1
@@ -198,6 +219,12 @@ def test_assign() -> None:
         dataclass_from_dict(_TestClass, {'ssval': {}})
     with pytest.raises(TypeError):
         dataclass_from_dict(_TestClass, {'ssval': set()})
+    with pytest.raises(TypeError):
+        dataclass_from_dict(_TestClass, {'tupleval': []})
+    with pytest.raises(TypeError):
+        dataclass_from_dict(_TestClass, {'tupleval': [1, 1, 1]})
+    with pytest.raises(TypeError):
+        dataclass_from_dict(_TestClass, {'tupleval': [2, 'foof', True, True]})
 
     # Fields with type Any should accept all types which are directly
     # supported by json, but not ones such as tuples or non-string dict keys
@@ -237,6 +264,14 @@ def test_assign() -> None:
     with pytest.raises(TypeError):
         dataclass_from_dict(_TestClass, {'fval': []}, coerce_to_float=True)
 
+    # Datetime values should only be allowed with timezone set as utc.
+    dataclass_to_dict(_TestClass(datetimeval=utc_now()))
+    with pytest.raises(ValueError):
+        dataclass_to_dict(_TestClass(datetimeval=datetime.datetime.now()))
+    with pytest.raises(ValueError):
+        # This doesn't actually set timezone on the datetime obj.
+        dataclass_to_dict(_TestClass(datetimeval=datetime.datetime.utcnow()))
+
 
 def test_coerce() -> None:
     """Test value coercion."""

tools/efro/dataclassio.py

@@ -8,6 +8,8 @@ unrecognized attribute data, allowing older clients to interact with newer
 data formats in a nondestructive manner.
 """
 
+# pylint: disable=too-many-lines
+
 # Note: We do lots of comparing of exact types here which is normally
 # frowned upon (stuff like isinstance() is usually encouraged).
 # pylint: disable=unidiomatic-typecheck
@@ -18,12 +20,22 @@ import logging
 from enum import Enum
 import dataclasses
 import typing
+import datetime
 from typing import TYPE_CHECKING, TypeVar, Generic, get_type_hints
 
 from efro.util import enum_by_value
 
+_pytz_utc: Any
+
+# We don't *require* pytz but we want to support it for tzinfos if available.
+try:
+    import pytz
+    _pytz_utc = pytz.utc
+except ModuleNotFoundError:
+    _pytz_utc = None  # pylint: disable=invalid-name
+
 if TYPE_CHECKING:
-    from typing import Any, Dict, Type, Tuple, Optional
+    from typing import Any, Dict, Type, Tuple, Optional, List
 
 T = TypeVar('T')
 
@@ -332,10 +344,35 @@ class PrepSession:
                                recursion_level=recursion_level + 1)
             return
 
+        # For Tuples, simply check individual member types.
+        # (and, for now, explicitly disallow zero member types or usage
+        # of ellipsis)
+        if origin is tuple:
+            childtypes = typing.get_args(anntype)
+            if not childtypes:
+                raise TypeError(
+                    f'Tuple at \'{attrname}\''
+                    f' has no type args; dataclassio requires type args.')
+            if childtypes[-1] is ...:
+                raise TypeError(f'Found ellipsis as part of type for'
+                                f' \'{attrname}\' on {cls}; these are not'
+                                f' supported by dataclassio.')
+            for childtype in childtypes:
+                self.prep_type(cls,
+                               attrname,
+                               childtype,
+                               recursion_level=recursion_level + 1)
+            return
+
         if issubclass(origin, Enum):
             self.prep_enum(origin)
             return
 
+        # We allow datetime objects (and google's extended subclass of them
+        # used in firestore, which is why we don't look for exact type here).
+        if issubclass(origin, datetime.datetime):
+            return
+
         if dataclasses.is_dataclass(origin):
             self.prep_dataclass(origin, recursion_level=recursion_level + 1)
             return
@@ -475,6 +512,7 @@ class _Outputter:
                        value: Any) -> Any:
         # pylint: disable=too-many-return-statements
         # pylint: disable=too-many-branches
+        # pylint: disable=too-many-statements
 
         origin = _get_origin(anntype)
 
@@ -513,6 +551,27 @@ class _Outputter:
                 _raise_type_error(fieldpath, type(value), (origin, ))
             return value if self._create else None
 
+        if origin is tuple:
+            if not isinstance(value, tuple):
+                raise TypeError(f'Expected a tuple for {fieldpath};'
+                                f' found a {type(value)}')
+            childanntypes = typing.get_args(anntype)
+
+            # We should have verified this was non-zero at prep-time
+            assert childanntypes
+            if len(value) != len(childanntypes):
+                raise TypeError(f'Tuple at {fieldpath} contains'
+                                f' {len(value)} values; type specifies'
+                                f' {len(childanntypes)}.')
+            if self._create:
+                return [
+                    self._process_value(cls, fieldpath, childanntypes[i], x)
+                    for i, x in enumerate(value)
+                ]
+            for i, x in enumerate(value):
+                self._process_value(cls, fieldpath, childanntypes[i], x)
+            return None
+
         if origin is list:
             if not isinstance(value, list):
                 raise TypeError(f'Expected a list for {fieldpath};'
@@ -585,6 +644,20 @@ class _Outputter:
             # types, so we can blindly return it here.
             return value.value if self._create else None
 
+        if issubclass(origin, datetime.datetime):
+            if not isinstance(value, origin):
+                raise TypeError(f'Expected a {origin} for {fieldpath};'
+                                f' found a {type(value)}.')
+            # We only support timezone-aware utc times.
+            if (value.tzinfo is not datetime.timezone.utc
+                    and (_pytz_utc is None or value.tzinfo is not _pytz_utc)):
+                raise ValueError(
+                    'datetime values must have timezone set as timezone.utc')
+            return [
+                value.year, value.month, value.day, value.hour, value.minute,
+                value.second, value.microsecond
+            ] if self._create else None
+
         raise TypeError(
             f"Field '{fieldpath}' of type '{anntype}' is unsupported here.")
 
@@ -677,6 +750,7 @@ class _Inputter(Generic[T]):
                           value: Any) -> Any:
         """Convert an assigned value to what a dataclass field expects."""
         # pylint: disable=too-many-return-statements
+        # pylint: disable=too-many-branches
 
         origin = _get_origin(anntype)
 
@@ -718,6 +792,9 @@ class _Inputter(Generic[T]):
             return self._sequence_from_input(cls, fieldpath, anntype, value,
                                              origin)
 
+        if origin is tuple:
+            return self._tuple_from_input(cls, fieldpath, anntype, value)
+
         if origin is dict:
             return self._dict_from_input(cls, fieldpath, anntype, value)
 
@@ -727,6 +804,9 @@ class _Inputter(Generic[T]):
         if issubclass(origin, Enum):
             return enum_by_value(origin, value)
 
+        if issubclass(origin, datetime.datetime):
+            return self._datetime_from_input(cls, fieldpath, value)
+
         raise TypeError(
             f"Field '{fieldpath}' of type '{anntype}' is unsupported here.")
 
@@ -901,3 +981,56 @@ class _Inputter(Generic[T]):
         return seqtype(
             self._value_from_input(cls, fieldpath, childanntype, i)
             for i in value)
+
+    def _datetime_from_input(self, cls: Type, fieldpath: str,
+                             value: Any) -> Any:
+
+        # We expect a list of 7 ints.
+        if type(value) is not list:
+            raise TypeError(
+                f'Invalid input value for "{fieldpath}" on "{cls}";'
+                f' expected a list, got a {type(value).__name__}')
+        if len(value) != 7 or not all(isinstance(x, int) for x in value):
+            raise TypeError(
+                f'Invalid input value for "{fieldpath}" on "{cls}";'
+                f' expected a list of 7 ints.')
+        return datetime.datetime(  # type: ignore
+            *value, tzinfo=datetime.timezone.utc)
+
+    def _tuple_from_input(self, cls: Type, fieldpath: str, anntype: Any,
+                          value: Any) -> Any:
+
+        out: List = []
+
+        # Because we are json-centric, we expect a list for all sequences.
+        if type(value) is not list:
+            raise TypeError(f'Invalid input value for "{fieldpath}";'
+                            f' expected a list, got a {type(value).__name__}')
+
+        childanntypes = typing.get_args(anntype)
+
+        # We should have verified this to be non-zero at prep-time.
+        assert childanntypes
+
+        if len(value) != len(childanntypes):
+            raise TypeError(f'Invalid tuple input for "{fieldpath}";'
+                            f' expected {len(childanntypes)} values,'
+                            f' found {len(value)}.')
+
+        for i, childanntype in enumerate(childanntypes):
+            childval = value[i]
+
+            # 'Any' type children; make sure they are valid json values
+            # and then just grab them.
+            if childanntype is typing.Any:
+                if not _is_valid_json(childval):
+                    raise TypeError(f'Item {i} of {fieldpath} contains'
+                                    f' data type(s) not supported by json.')
+                out.append(childval)
+            else:
+                out.append(
+                    self._value_from_input(cls, fieldpath, childanntype,
+                                           childval))
+
+        assert len(out) == len(childanntypes)
+        return tuple(out)