diff --git a/.efrocachemap b/.efrocachemap index 2ea16a71..04d00178 100644 --- a/.efrocachemap +++ b/.efrocachemap @@ -3940,18 +3940,18 @@ "build/prefab/full/linux_x86_64/release/ballisticacore": "https://files.ballistica.net/cache/ba1/8b/8b/b8f8a75b3ded113231265f61da9d", "build/prefab/full/linux_x86_64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/14/4e/bd10863753f44c7612ef697c4693", "build/prefab/full/linux_x86_64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/34/a0/883c84cb130780bb8bc8a2185604", - "build/prefab/full/mac_arm64/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/d8/73/ef08a58076c8fc37af28bf097628", + "build/prefab/full/mac_arm64/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/e0/ee/e6b94bf4149530e412cfe27e5cb4", "build/prefab/full/mac_arm64/release/ballisticacore": "https://files.ballistica.net/cache/ba1/97/62/0944cd3ab34d681cd7c9bfaa0b11", - "build/prefab/full/mac_arm64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/9c/c0/ab01fbc7544f451db865b93e5430", + "build/prefab/full/mac_arm64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/dd/a5/c73d18aba833987ff3e713bbf981", "build/prefab/full/mac_arm64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/27/e1/de75aac52e10bebae81fc12aa030", - "build/prefab/full/mac_x86_64/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/d0/53/f358c93ad50992c3aee613034e38", + "build/prefab/full/mac_x86_64/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/a6/81/ee957a5bfd1be45c0e865f8a27ba", "build/prefab/full/mac_x86_64/release/ballisticacore": "https://files.ballistica.net/cache/ba1/03/29/abffbfc56fd981915253f1d4ed96", - "build/prefab/full/mac_x86_64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/a4/83/bf81025600a3a440d359057f0e05", + "build/prefab/full/mac_x86_64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/04/35/104446e5f91b9fe35fa413be02ca", "build/prefab/full/mac_x86_64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/ed/c0/68b44a693639308981b5214f02c1", - "build/prefab/full/windows_x86/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/5b/f8/e1934b7fd16e75bdf9dadccef22b", - "build/prefab/full/windows_x86/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/1d/ad/193fc3816804c48954a6b2901b49", - "build/prefab/full/windows_x86_server/debug/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/89/c5/0fcd0aafe25cb054f165b3ede0e5", - "build/prefab/full/windows_x86_server/release/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/ae/2c/bfa29fcf8e83db1cc5bb32da1178", + "build/prefab/full/windows_x86/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/b5/c4/5935bc59cc237c42e8ac9be47ce5", + "build/prefab/full/windows_x86/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/35/3d/f04ebd4a7d066088595b8ed4bbc5", + "build/prefab/full/windows_x86_server/debug/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/72/eb/25cf435771601aeec273a92ffb50", + "build/prefab/full/windows_x86_server/release/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/45/fb/166e932a6235613935ccf2e51d00", "build/prefab/lib/linux_arm64/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/f4/59/16c01f646c16bb480a197aa9e6e3", "build/prefab/lib/linux_arm64/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/81/73/a321fe4aa721d07f2a44257967cf", "build/prefab/lib/linux_arm64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/99/99/2adf5a22923eed3f8b5667ee8220", @@ -3960,12 +3960,12 @@ "build/prefab/lib/linux_x86_64/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/3d/3a/c747993afbf4c1ed1c5e8b0e5d5a", "build/prefab/lib/linux_x86_64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/46/21/31fe300afbf7d9da766c04064919", "build/prefab/lib/linux_x86_64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/f8/bd/2b05dbfd98cd55cedf924b639765", - "build/prefab/lib/mac_arm64/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/a9/1d/3896630323d201928a9a94d1132b", - "build/prefab/lib/mac_arm64/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/39/eb/512862ff8b6e5aca44d4ef9b2b00", - "build/prefab/lib/mac_arm64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/7c/db/2644a5447ec880608cc719211a03", - "build/prefab/lib/mac_arm64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/7a/d6/fe10f5e34d8c86a47ddbea041297", - "build/prefab/lib/mac_x86_64/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/f1/f8/0529ec1e41b71f9e595eff5d9655", - "build/prefab/lib/mac_x86_64/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/b8/e5/4acf3b0f3031a22ef02a8ec32a3d", - "build/prefab/lib/mac_x86_64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/a4/c5/1ec7762968c5ab8a95d4b2203310", - "build/prefab/lib/mac_x86_64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/84/23/0a99eea99bcf009a1801eb6c4755" + "build/prefab/lib/mac_arm64/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/5b/6d/877a5a015eb3d705795a2db7fa74", + "build/prefab/lib/mac_arm64/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/58/a4/3d2a1e4a47c51e883ef29b5e280d", + "build/prefab/lib/mac_arm64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/a7/ca/91cfd7dc2ccc8c996daf9aaeb9fd", + "build/prefab/lib/mac_arm64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/93/cb/de559dd0ea55e12a228ea1cd8974", + "build/prefab/lib/mac_x86_64/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/c9/e1/132123edad385ac0a977337f044c", + "build/prefab/lib/mac_x86_64/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/f2/e3/a82f88651f2ffef8c52a20da42a6", + "build/prefab/lib/mac_x86_64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/d4/9c/95e358c6e929bcb046e40fe5ad2b", + "build/prefab/lib/mac_x86_64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/8e/2d/81ce3feefa50c283cd7a8398ac72" } \ No newline at end of file diff --git a/.idea/dictionaries/ericf.xml b/.idea/dictionaries/ericf.xml index b57e22d1..7e939da7 100644 --- a/.idea/dictionaries/ericf.xml +++ b/.idea/dictionaries/ericf.xml @@ -345,6 +345,7 @@ childnode childtype childtypes + childval chinesetraditional chipfork chosenone @@ -491,6 +492,7 @@ dataval datetimemodule datetimes + datetimeval daynum dayoffset dbapi @@ -2283,6 +2285,7 @@ tself tspc tstr + tupleval turtledemo tval tvalue diff --git a/Makefile b/Makefile index 320c31da..2aa2bfd1 100644 --- a/Makefile +++ b/Makefile @@ -661,7 +661,7 @@ test-assetmanager: tests/test_ba/test_assetmanager.py::test_assetmanager # Individual test with extra output enabled. -test-dataclasses: +test-dataclassio: @tools/pcommand pytest -o log_cli=true -o log_cli_level=debug -s -vv \ tests/test_efro/test_dataclassio.py diff --git a/ballisticacore-cmake/.idea/dictionaries/ericf.xml b/ballisticacore-cmake/.idea/dictionaries/ericf.xml index 59f7b267..c416647c 100644 --- a/ballisticacore-cmake/.idea/dictionaries/ericf.xml +++ b/ballisticacore-cmake/.idea/dictionaries/ericf.xml @@ -164,6 +164,7 @@ childanntypes childtype childtypes + childval chrono chunksize cjief @@ -220,6 +221,7 @@ datas datav datavec + datetimeval dbgstr dbias dcioexattrs @@ -995,6 +997,7 @@ trimeshes trynum tself + tupleval tval tvos tweakage diff --git a/docs/ba_module.md b/docs/ba_module.md index 86248abc..ea0c52d8 100644 --- a/docs/ba_module.md +++ b/docs/ba_module.md @@ -1,5 +1,5 @@ -

last updated on 2021-04-30 for Ballistica version 1.6.0 build 20353

+

last updated on 2021-05-03 for Ballistica version 1.6.0 build 20355

This page documents the Python classes and functions in the 'ba' module, which are the ones most relevant to modding in Ballistica. If you come across something you feel should be included here or could be better explained, please let me know. Happy modding!

diff --git a/tests/test_efro/test_dataclassio.py b/tests/test_efro/test_dataclassio.py index cbcaf10f..43fed197 100644 --- a/tests/test_efro/test_dataclassio.py +++ b/tests/test_efro/test_dataclassio.py @@ -5,16 +5,18 @@ from __future__ import annotations from enum import Enum +import datetime from dataclasses import field, dataclass from typing import TYPE_CHECKING import pytest +from efro.util import utc_now from efro.dataclassio import (dataclass_validate, dataclass_from_dict, dataclass_to_dict, prepped) if TYPE_CHECKING: - from typing import Optional, List, Set, Any, Dict, Sequence, Union + from typing import Optional, List, Set, Any, Dict, Sequence, Union, Tuple class _EnumTest(Enum): @@ -75,6 +77,8 @@ def test_assign() -> None: ssval: Set[str] = field(default_factory=set) anyval: Any = 1 dictval: Dict[int, str] = field(default_factory=dict) + tupleval: Tuple[int, str, bool] = (1, 'foo', False) + datetimeval: Optional[datetime.datetime] = None class _TestClass2: pass @@ -89,14 +93,20 @@ def test_assign() -> None: with pytest.raises(TypeError): dataclass_from_dict(_TestClass, None) # type: ignore + now = utc_now() + # A dict containing *ALL* values should match what we # get when creating a dataclass and then converting back # to a dict. dict1 = { - 'ival': 1, - 'sval': 'foo', - 'bval': True, - 'fval': 2.0, + 'ival': + 1, + 'sval': + 'foo', + 'bval': + True, + 'fval': + 2.0, 'nval': { 'ival': 1, 'sval': 'bar', @@ -104,12 +114,18 @@ def test_assign() -> None: '1': 'foof' }, }, - 'enval': 'test1', - 'oival': 1, - 'osval': 'foo', - 'obval': True, - 'ofval': 1.0, - 'oenval': 'test2', + 'enval': + 'test1', + 'oival': + 1, + 'osval': + 'foo', + 'obval': + True, + 'ofval': + 1.0, + 'oenval': + 'test2', 'lsval': ['foo'], 'lival': [10], 'lbval': [False], @@ -127,7 +143,12 @@ def test_assign() -> None: }, 'dictval': { '1': 'foo' - } + }, + 'tupleval': [2, 'foof', True], + 'datetimeval': [ + now.year, now.month, now.day, now.hour, now.minute, now.second, + now.microsecond + ], } dc1 = dataclass_from_dict(_TestClass, dict1) assert dataclass_to_dict(dc1) == dict1 @@ -198,6 +219,12 @@ def test_assign() -> None: dataclass_from_dict(_TestClass, {'ssval': {}}) with pytest.raises(TypeError): dataclass_from_dict(_TestClass, {'ssval': set()}) + with pytest.raises(TypeError): + dataclass_from_dict(_TestClass, {'tupleval': []}) + with pytest.raises(TypeError): + dataclass_from_dict(_TestClass, {'tupleval': [1, 1, 1]}) + with pytest.raises(TypeError): + dataclass_from_dict(_TestClass, {'tupleval': [2, 'foof', True, True]}) # Fields with type Any should accept all types which are directly # supported by json, but not ones such as tuples or non-string dict keys @@ -237,6 +264,14 @@ def test_assign() -> None: with pytest.raises(TypeError): dataclass_from_dict(_TestClass, {'fval': []}, coerce_to_float=True) + # Datetime values should only be allowed with timezone set as utc. + dataclass_to_dict(_TestClass(datetimeval=utc_now())) + with pytest.raises(ValueError): + dataclass_to_dict(_TestClass(datetimeval=datetime.datetime.now())) + with pytest.raises(ValueError): + # This doesn't actually set timezone on the datetime obj. + dataclass_to_dict(_TestClass(datetimeval=datetime.datetime.utcnow())) + def test_coerce() -> None: """Test value coercion.""" diff --git a/tools/efro/dataclassio.py b/tools/efro/dataclassio.py index b76e641a..4b1fed96 100644 --- a/tools/efro/dataclassio.py +++ b/tools/efro/dataclassio.py @@ -8,6 +8,8 @@ unrecognized attribute data, allowing older clients to interact with newer data formats in a nondestructive manner. """ +# pylint: disable=too-many-lines + # Note: We do lots of comparing of exact types here which is normally # frowned upon (stuff like isinstance() is usually encouraged). # pylint: disable=unidiomatic-typecheck @@ -18,12 +20,22 @@ import logging from enum import Enum import dataclasses import typing +import datetime from typing import TYPE_CHECKING, TypeVar, Generic, get_type_hints from efro.util import enum_by_value +_pytz_utc: Any + +# We don't *require* pytz but we want to support it for tzinfos if available. +try: + import pytz + _pytz_utc = pytz.utc +except ModuleNotFoundError: + _pytz_utc = None # pylint: disable=invalid-name + if TYPE_CHECKING: - from typing import Any, Dict, Type, Tuple, Optional + from typing import Any, Dict, Type, Tuple, Optional, List T = TypeVar('T') @@ -332,10 +344,35 @@ class PrepSession: recursion_level=recursion_level + 1) return + # For Tuples, simply check individual member types. + # (and, for now, explicitly disallow zero member types or usage + # of ellipsis) + if origin is tuple: + childtypes = typing.get_args(anntype) + if not childtypes: + raise TypeError( + f'Tuple at \'{attrname}\'' + f' has no type args; dataclassio requires type args.') + if childtypes[-1] is ...: + raise TypeError(f'Found ellipsis as part of type for' + f' \'{attrname}\' on {cls}; these are not' + f' supported by dataclassio.') + for childtype in childtypes: + self.prep_type(cls, + attrname, + childtype, + recursion_level=recursion_level + 1) + return + if issubclass(origin, Enum): self.prep_enum(origin) return + # We allow datetime objects (and google's extended subclass of them + # used in firestore, which is why we don't look for exact type here). + if issubclass(origin, datetime.datetime): + return + if dataclasses.is_dataclass(origin): self.prep_dataclass(origin, recursion_level=recursion_level + 1) return @@ -475,6 +512,7 @@ class _Outputter: value: Any) -> Any: # pylint: disable=too-many-return-statements # pylint: disable=too-many-branches + # pylint: disable=too-many-statements origin = _get_origin(anntype) @@ -513,6 +551,27 @@ class _Outputter: _raise_type_error(fieldpath, type(value), (origin, )) return value if self._create else None + if origin is tuple: + if not isinstance(value, tuple): + raise TypeError(f'Expected a tuple for {fieldpath};' + f' found a {type(value)}') + childanntypes = typing.get_args(anntype) + + # We should have verified this was non-zero at prep-time + assert childanntypes + if len(value) != len(childanntypes): + raise TypeError(f'Tuple at {fieldpath} contains' + f' {len(value)} values; type specifies' + f' {len(childanntypes)}.') + if self._create: + return [ + self._process_value(cls, fieldpath, childanntypes[i], x) + for i, x in enumerate(value) + ] + for i, x in enumerate(value): + self._process_value(cls, fieldpath, childanntypes[i], x) + return None + if origin is list: if not isinstance(value, list): raise TypeError(f'Expected a list for {fieldpath};' @@ -585,6 +644,20 @@ class _Outputter: # types, so we can blindly return it here. return value.value if self._create else None + if issubclass(origin, datetime.datetime): + if not isinstance(value, origin): + raise TypeError(f'Expected a {origin} for {fieldpath};' + f' found a {type(value)}.') + # We only support timezone-aware utc times. + if (value.tzinfo is not datetime.timezone.utc + and (_pytz_utc is None or value.tzinfo is not _pytz_utc)): + raise ValueError( + 'datetime values must have timezone set as timezone.utc') + return [ + value.year, value.month, value.day, value.hour, value.minute, + value.second, value.microsecond + ] if self._create else None + raise TypeError( f"Field '{fieldpath}' of type '{anntype}' is unsupported here.") @@ -677,6 +750,7 @@ class _Inputter(Generic[T]): value: Any) -> Any: """Convert an assigned value to what a dataclass field expects.""" # pylint: disable=too-many-return-statements + # pylint: disable=too-many-branches origin = _get_origin(anntype) @@ -718,6 +792,9 @@ class _Inputter(Generic[T]): return self._sequence_from_input(cls, fieldpath, anntype, value, origin) + if origin is tuple: + return self._tuple_from_input(cls, fieldpath, anntype, value) + if origin is dict: return self._dict_from_input(cls, fieldpath, anntype, value) @@ -727,6 +804,9 @@ class _Inputter(Generic[T]): if issubclass(origin, Enum): return enum_by_value(origin, value) + if issubclass(origin, datetime.datetime): + return self._datetime_from_input(cls, fieldpath, value) + raise TypeError( f"Field '{fieldpath}' of type '{anntype}' is unsupported here.") @@ -901,3 +981,56 @@ class _Inputter(Generic[T]): return seqtype( self._value_from_input(cls, fieldpath, childanntype, i) for i in value) + + def _datetime_from_input(self, cls: Type, fieldpath: str, + value: Any) -> Any: + + # We expect a list of 7 ints. + if type(value) is not list: + raise TypeError( + f'Invalid input value for "{fieldpath}" on "{cls}";' + f' expected a list, got a {type(value).__name__}') + if len(value) != 7 or not all(isinstance(x, int) for x in value): + raise TypeError( + f'Invalid input value for "{fieldpath}" on "{cls}";' + f' expected a list of 7 ints.') + return datetime.datetime( # type: ignore + *value, tzinfo=datetime.timezone.utc) + + def _tuple_from_input(self, cls: Type, fieldpath: str, anntype: Any, + value: Any) -> Any: + + out: List = [] + + # Because we are json-centric, we expect a list for all sequences. + if type(value) is not list: + raise TypeError(f'Invalid input value for "{fieldpath}";' + f' expected a list, got a {type(value).__name__}') + + childanntypes = typing.get_args(anntype) + + # We should have verified this to be non-zero at prep-time. + assert childanntypes + + if len(value) != len(childanntypes): + raise TypeError(f'Invalid tuple input for "{fieldpath}";' + f' expected {len(childanntypes)} values,' + f' found {len(value)}.') + + for i, childanntype in enumerate(childanntypes): + childval = value[i] + + # 'Any' type children; make sure they are valid json values + # and then just grab them. + if childanntype is typing.Any: + if not _is_valid_json(childval): + raise TypeError(f'Item {i} of {fieldpath} contains' + f' data type(s) not supported by json.') + out.append(childval) + else: + out.append( + self._value_from_input(cls, fieldpath, childanntype, + childval)) + + assert len(out) == len(childanntypes) + return tuple(out)