RYDE-WORK/ballistica
1
0
Fork 0
mirror of https://github.com/RYDE-WORK/ballistica.git synced 2026-02-07 08:03:30 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Improved dataclassio handling of 'Any' types

Browse Source
This commit is contained in:
Eric Froemling 2021-05-27 14:32:52 -05:00
parent 06514e913b
commit 8843b8cccd
No known key found for this signature in database
GPG Key ID: 89C93F0F8D6D5A98
2 changed files with 116 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
tests/test_efro/test_dataclassio.py
View File

@ -654,6 +654,32 @@ def test_name_clashes() -> None:
ival2: Annotated[int, IOAttrs('ival')] = 5 ival2: Annotated[int, IOAttrs('ival')] = 5
def test_any() -> None:
"""Test data included with type Any."""
@ioprepped
@dataclass
class _TestClass:
anyval: Any
obj = _TestClass(anyval=b'bytes')
# JSON output doesn't allow bytes or datetime objects
# included in 'Any' data.
with pytest.raises(TypeError):
dataclass_validate(obj, codec=Codec.JSON)
obj.anyval = datetime.datetime.now()
with pytest.raises(TypeError):
dataclass_validate(obj, codec=Codec.JSON)
# Firestore, however, does.
obj.anyval = b'bytes'
dataclass_validate(obj, codec=Codec.FIRESTORE)
obj.anyval = datetime.datetime.now()
dataclass_validate(obj, codec=Codec.FIRESTORE)
@ioprepped @ioprepped
@dataclass @dataclass
class _SPTestClass1: class _SPTestClass1:

153
tools/efro/dataclassio.py
View File

@ -474,7 +474,7 @@ class PrepSession:
else: else:
raise TypeError( raise TypeError(
f'Dict key type {childtypes[0]} for \'{attrname}\'' f'Dict key type {childtypes[0]} for \'{attrname}\''
f' on {cls} is not supported by dataclassio.') f' on {cls.__name__} is not supported by dataclassio.')
# For value types we support any of our normal types. # For value types we support any of our normal types.
if not childtypes or _get_origin(childtypes[1]) is typing.Any: if not childtypes or _get_origin(childtypes[1]) is typing.Any:
@ -498,7 +498,8 @@ class PrepSession:
f' has no type args; dataclassio requires type args.') f' has no type args; dataclassio requires type args.')
if childtypes[-1] is ...: if childtypes[-1] is ...:
raise TypeError(f'Found ellipsis as part of type for' raise TypeError(f'Found ellipsis as part of type for'
f' \'{attrname}\' on {cls}; these are not' f' \'{attrname}\' on {cls.__name__};'
f' these are not'
f' supported by dataclassio.') f' supported by dataclassio.')
for childtype in childtypes: for childtype in childtypes:
self.prep_type(cls, self.prep_type(cls,
@ -523,7 +524,8 @@ class PrepSession:
if origin is bytes: if origin is bytes:
return return
raise TypeError(f"Attr '{attrname}' on {cls} contains type '{anntype}'" raise TypeError(f"Attr '{attrname}' on {cls.__name__} contains"
f" type '{anntype}'"
f' which is not supported by dataclassio.') f' which is not supported by dataclassio.')
def prep_union(self, cls: Type, attrname: str, anntype: Any, def prep_union(self, cls: Type, attrname: str, anntype: Any,
@ -533,7 +535,7 @@ class PrepSession:
if (len(typeargs) != 2 if (len(typeargs) != 2
or len([c for c in typeargs if c is type(None)]) != 1): or len([c for c in typeargs if c is type(None)]) != 1):
raise TypeError(f'Union {anntype} for attr \'{attrname}\' on' raise TypeError(f'Union {anntype} for attr \'{attrname}\' on'
f' {cls} is not supported by dataclassio;' f' {cls.__name__} is not supported by dataclassio;'
f' only 2 member Unions with one type being None' f' only 2 member Unions with one type being None'
f' are supported.') f' are supported.')
for childtype in typeargs: for childtype in typeargs:
@ -563,7 +565,7 @@ class PrepSession:
f' them to be uniform.') f' them to be uniform.')
def _is_valid_json(obj: Any) -> bool: def _is_valid_for_codec(obj: Any, codec: Codec) -> bool:
"""Return whether a value consists solely of json-supported types. """Return whether a value consists solely of json-supported types.
Note that this does not include things like tuples which are Note that this does not include things like tuples which are
@ -578,9 +580,15 @@ def _is_valid_json(obj: Any) -> bool:
if objtype is dict: if objtype is dict:
# JSON 'objects' supports only string dict keys, but all value types. # JSON 'objects' supports only string dict keys, but all value types.
return all( return all(
type(k) is str and _is_valid_json(v) for k, v in obj.items()) type(k) is str and _is_valid_for_codec(v, codec)
for k, v in obj.items())
if objtype is list: if objtype is list:
return all(_is_valid_json(elem) for elem in obj) return all(_is_valid_for_codec(elem, codec) for elem in obj)
# A few things are valid in firestore but not json.
if issubclass(objtype, datetime.datetime) or objtype is bytes:
return codec is Codec.FIRESTORE
return False return False
@ -655,7 +663,7 @@ class _Outputter:
continue continue
else: else:
raise RuntimeError( raise RuntimeError(
f'Field {fieldname} of {cls} has' f'Field {fieldname} of {cls.__name__} has'
f' neither a default nor a default_factory;' f' neither a default nor a default_factory;'
f' store_default=False cannot be set for it.' f' store_default=False cannot be set for it.'
f' (AND THIS SHOULD HAVE BEEN CAUGHT IN PREP!)') f' (AND THIS SHOULD HAVE BEEN CAUGHT IN PREP!)')
@ -672,7 +680,7 @@ class _Outputter:
# If there's extra-attrs stored on us, check/include them. # If there's extra-attrs stored on us, check/include them.
extra_attrs = getattr(obj, EXTRA_ATTRS_ATTR, None) extra_attrs = getattr(obj, EXTRA_ATTRS_ATTR, None)
if isinstance(extra_attrs, dict): if isinstance(extra_attrs, dict):
if not _is_valid_json(extra_attrs): if not _is_valid_for_codec(extra_attrs, self._codec):
raise TypeError( raise TypeError(
f'Extra attrs on {fieldpath} contains data type(s)' f'Extra attrs on {fieldpath} contains data type(s)'
f' not supported by json.') f' not supported by json.')
@ -690,11 +698,12 @@ class _Outputter:
origin = _get_origin(anntype) origin = _get_origin(anntype)
if origin is typing.Any: if origin is typing.Any:
if not _is_valid_json(value): if not _is_valid_for_codec(value, self._codec):
raise TypeError(f'Invalid value type for \'{fieldpath}\';' raise TypeError(
f" 'Any' typed values must be types directly" f'Invalid value type for \'{fieldpath}\';'
f' supported by json; got' f" 'Any' typed values must contain types directly"
f" '{type(value).__name__}'.") f' supported by the specified codec ({self._codec.name});'
f' found \'{type(value).__name__}\' which is not.')
return value if self._create else None return value if self._create else None
if origin is typing.Union: if origin is typing.Union:
@ -752,13 +761,15 @@ class _Outputter:
f' found a {type(value)}') f' found a {type(value)}')
childanntypes = typing.get_args(anntype) childanntypes = typing.get_args(anntype)
# 'Any' type children; make sure they are valid json values. # 'Any' type children; make sure they are valid values for
# the specified codec.
if len(childanntypes) == 0 or childanntypes[0] is typing.Any: if len(childanntypes) == 0 or childanntypes[0] is typing.Any:
for i, child in enumerate(value): for i, child in enumerate(value):
if not _is_valid_json(child): if not _is_valid_for_codec(child, self._codec):
raise TypeError( raise TypeError(
f'Item {i} of {fieldpath} contains' f'Item {i} of {fieldpath} contains'
f' data type(s) not supported by json.') f' data type(s) not supported by the specified'
f' codec ({self._codec.name}).')
# Hmm; should we do a copy here? # Hmm; should we do a copy here?
return value if self._create else None return value if self._create else None
@ -783,10 +794,11 @@ class _Outputter:
# 'Any' type children; make sure they are valid Any values. # 'Any' type children; make sure they are valid Any values.
if len(childanntypes) == 0 or childanntypes[0] is typing.Any: if len(childanntypes) == 0 or childanntypes[0] is typing.Any:
for child in value: for child in value:
if not _is_valid_json(child): if not _is_valid_for_codec(child, self._codec):
raise TypeError( raise TypeError(
f'Set at {fieldpath} contains' f'Set at {fieldpath} contains'
f' data type(s) not supported by json.') f' data type(s) not supported by the'
f' specified codec ({self._codec.name}).')
return list(value) if self._create else None return list(value) if self._create else None
# We contain elements of some specified type. # We contain elements of some specified type.
@ -844,8 +856,9 @@ class _Outputter:
def _process_bytes(self, cls: Type, fieldpath: str, value: bytes) -> Any: def _process_bytes(self, cls: Type, fieldpath: str, value: bytes) -> Any:
import base64 import base64
if not isinstance(value, bytes): if not isinstance(value, bytes):
raise TypeError(f'Expected bytes for {fieldpath} on {cls};' raise TypeError(
f' found a {type(value)}.') f'Expected bytes for {fieldpath} on {cls.__name__};'
f' found a {type(value)}.')
if not self._create: if not self._create:
return None return None
@ -868,11 +881,14 @@ class _Outputter:
# We treat 'Any' dicts simply as json; we don't do any translating. # We treat 'Any' dicts simply as json; we don't do any translating.
if not childtypes or childtypes[0] is typing.Any: if not childtypes or childtypes[0] is typing.Any:
if not isinstance(value, dict) or not _is_valid_json(value): if not isinstance(value, dict) or not _is_valid_for_codec(
value, self._codec):
raise TypeError( raise TypeError(
f'Invalid value for Dict[Any, Any]' f'Invalid value for Dict[Any, Any]'
f' at \'{fieldpath}\' on {cls}; all keys and values' f' at \'{fieldpath}\' on {cls.__name__};'
f' must be json-compatible when dict type is Any.') f' all keys and values must be directly compatible'
f' with the specified codec ({self._codec.name})'
f' when dict type is Any.')
return value if self._create else None return value if self._create else None
# Ok; we've got a definite key type (which we verified as valid # Ok; we've got a definite key type (which we verified as valid
@ -884,22 +900,24 @@ class _Outputter:
if keyanntype is str: if keyanntype is str:
for key, val in value.items(): for key, val in value.items():
if not isinstance(key, str): if not isinstance(key, str):
raise TypeError(f'Got invalid key type {type(key)} for' raise TypeError(
f' dict key at \'{fieldpath}\' on {cls};' f'Got invalid key type {type(key)} for'
f' expected {keyanntype}.') f' dict key at \'{fieldpath}\' on {cls.__name__};'
f' expected {keyanntype}.')
outval = self._process_value(cls, fieldpath, valanntype, val, outval = self._process_value(cls, fieldpath, valanntype, val,
ioattrs) ioattrs)
if self._create: if self._create:
assert out is not None assert out is not None
out[key] = outval out[key] = outval
# int keys are stored in json as str versions of themselves. # int keys are stored as str versions of themselves.
elif keyanntype is int: elif keyanntype is int:
for key, val in value.items(): for key, val in value.items():
if not isinstance(key, int): if not isinstance(key, int):
raise TypeError(f'Got invalid key type {type(key)} for' raise TypeError(
f' dict key at \'{fieldpath}\' on {cls};' f'Got invalid key type {type(key)} for'
f' expected an int.') f' dict key at \'{fieldpath}\' on {cls.__name__};'
f' expected an int.')
outval = self._process_value(cls, fieldpath, valanntype, val, outval = self._process_value(cls, fieldpath, valanntype, val,
ioattrs) ioattrs)
if self._create: if self._create:
@ -909,9 +927,10 @@ class _Outputter:
elif issubclass(keyanntype, Enum): elif issubclass(keyanntype, Enum):
for key, val in value.items(): for key, val in value.items():
if not isinstance(key, keyanntype): if not isinstance(key, keyanntype):
raise TypeError(f'Got invalid key type {type(key)} for' raise TypeError(
f' dict key at \'{fieldpath}\' on {cls};' f'Got invalid key type {type(key)} for'
f' expected a {keyanntype}.') f' dict key at \'{fieldpath}\' on {cls.__name__};'
f' expected a {keyanntype}.')
outval = self._process_value(cls, fieldpath, valanntype, val, outval = self._process_value(cls, fieldpath, valanntype, val,
ioattrs) ioattrs)
if self._create: if self._create:
@ -956,11 +975,12 @@ class _Inputter(Generic[T]):
origin = _get_origin(anntype) origin = _get_origin(anntype)
if origin is typing.Any: if origin is typing.Any:
if not _is_valid_json(value): if not _is_valid_for_codec(value, self._codec):
raise TypeError(f'Invalid value type for \'{fieldpath}\';' raise TypeError(f'Invalid value type for \'{fieldpath}\';'
f' \'Any\' typed values must be types directly' f' \'Any\' typed values must contain only'
f' supported by json; got' f' types directly supported by the specified'
f' \'{type(value).__name__}\'.') f' codec ({self._codec.name}); found'
f' \'{type(value).__name__}\' which is not.')
return value return value
if origin is typing.Union: if origin is typing.Union:
@ -1026,14 +1046,14 @@ class _Inputter(Generic[T]):
if self._codec is Codec.FIRESTORE: if self._codec is Codec.FIRESTORE:
if not isinstance(value, bytes): if not isinstance(value, bytes):
raise TypeError(f'Expected a bytes object for {fieldpath}' raise TypeError(f'Expected a bytes object for {fieldpath}'
f' on {cls}; got a {type(value)}.') f' on {cls.__name__}; got a {type(value)}.')
return value return value
assert self._codec is Codec.JSON assert self._codec is Codec.JSON
if not isinstance(value, str): if not isinstance(value, str):
raise TypeError(f'Expected a string object for {fieldpath}' raise TypeError(f'Expected a string object for {fieldpath}'
f' on {cls}; got a {type(value)}.') f' on {cls.__name__}; got a {type(value)}.')
return base64.b64decode(value) return base64.b64decode(value)
def _dataclass_from_input(self, cls: Type, fieldpath: str, def _dataclass_from_input(self, cls: Type, fieldpath: str,
@ -1047,8 +1067,9 @@ class _Inputter(Generic[T]):
""" """
# pylint: disable=too-many-locals # pylint: disable=too-many-locals
if not isinstance(values, dict): if not isinstance(values, dict):
raise TypeError(f'Expected a dict for {fieldpath} on {cls};' raise TypeError(
f' got a {type(values)}.') f'Expected a dict for {fieldpath} on {cls.__name__};'
f' got a {type(values)}.')
prep = PrepSession(explicit=False).prep_dataclass(cls, prep = PrepSession(explicit=False).prep_dataclass(cls,
recursion_level=0) recursion_level=0)
@ -1071,11 +1092,12 @@ class _Inputter(Generic[T]):
# Treat this like 'Any' data; ensure that it is valid # Treat this like 'Any' data; ensure that it is valid
# raw json. # raw json.
if not _is_valid_json(value): if not _is_valid_for_codec(value, self._codec):
raise TypeError( raise TypeError(
f'Unknown attr {key}' f'Unknown attr \'{key}\''
f' on {fieldpath} contains data type(s)' f' on {fieldpath} contains data type(s)'
f' not supported by json.') f' not supported by the specified codec'
f' ({self._codec.name}).')
extra_attrs[key] = value extra_attrs[key] = value
else: else:
raise AttributeError( raise AttributeError(
@ -1092,9 +1114,8 @@ class _Inputter(Generic[T]):
try: try:
out = cls(**args) out = cls(**args)
except Exception as exc: except Exception as exc:
raise RuntimeError( raise RuntimeError(f'Error instantiating class {cls.__name__}'
f'Error instantiating class {cls} at {fieldpath}: {exc}' f' at {fieldpath}: {exc}') from exc
) from exc
if extra_attrs: if extra_attrs:
setattr(out, EXTRA_ATTRS_ATTR, extra_attrs) setattr(out, EXTRA_ATTRS_ATTR, extra_attrs)
return out return out
@ -1105,8 +1126,9 @@ class _Inputter(Generic[T]):
# pylint: disable=too-many-locals # pylint: disable=too-many-locals
if not isinstance(value, dict): if not isinstance(value, dict):
raise TypeError(f'Expected a dict for \'{fieldpath}\' on {cls};' raise TypeError(
f' got a {type(value)}.') f'Expected a dict for \'{fieldpath}\' on {cls.__name__};'
f' got a {type(value)}.')
childtypes = typing.get_args(anntype) childtypes = typing.get_args(anntype)
assert len(childtypes) in (0, 2) assert len(childtypes) in (0, 2)
@ -1115,11 +1137,13 @@ class _Inputter(Generic[T]):
# We treat 'Any' dicts simply as json; we don't do any translating. # We treat 'Any' dicts simply as json; we don't do any translating.
if not childtypes or childtypes[0] is typing.Any: if not childtypes or childtypes[0] is typing.Any:
if not isinstance(value, dict) or not _is_valid_json(value): if not isinstance(value, dict) or not _is_valid_for_codec(
value, self._codec):
raise TypeError(f'Got invalid value for Dict[Any, Any]' raise TypeError(f'Got invalid value for Dict[Any, Any]'
f' at \'{fieldpath}\' on {cls};' f' at \'{fieldpath}\' on {cls.__name__};'
f' all keys and values must be' f' all keys and values must be'
f' json-compatible.') f' compatible with the specified codec'
f' ({self._codec.name}).')
out = value out = value
else: else:
out = {} out = {}
@ -1134,7 +1158,7 @@ class _Inputter(Generic[T]):
if not isinstance(key, str): if not isinstance(key, str):
raise TypeError( raise TypeError(
f'Got invalid key type {type(key)} for' f'Got invalid key type {type(key)} for'
f' dict key at \'{fieldpath}\' on {cls};' f' dict key at \'{fieldpath}\' on {cls.__name__};'
f' expected a str.') f' expected a str.')
out[key] = self._value_from_input(cls, fieldpath, out[key] = self._value_from_input(cls, fieldpath,
valanntype, val, ioattrs) valanntype, val, ioattrs)
@ -1145,14 +1169,14 @@ class _Inputter(Generic[T]):
if not isinstance(key, str): if not isinstance(key, str):
raise TypeError( raise TypeError(
f'Got invalid key type {type(key)} for' f'Got invalid key type {type(key)} for'
f' dict key at \'{fieldpath}\' on {cls};' f' dict key at \'{fieldpath}\' on {cls.__name__};'
f' expected a str.') f' expected a str.')
try: try:
keyint = int(key) keyint = int(key)
except ValueError as exc: except ValueError as exc:
raise TypeError( raise TypeError(
f'Got invalid key value {key} for' f'Got invalid key value {key} for'
f' dict key at \'{fieldpath}\' on {cls};' f' dict key at \'{fieldpath}\' on {cls.__name__};'
f' expected an int in string form.') from exc f' expected an int in string form.') from exc
out[keyint] = self._value_from_input( out[keyint] = self._value_from_input(
cls, fieldpath, valanntype, val, ioattrs) cls, fieldpath, valanntype, val, ioattrs)
@ -1170,7 +1194,8 @@ class _Inputter(Generic[T]):
except ValueError as exc: except ValueError as exc:
raise ValueError( raise ValueError(
f'Got invalid key value {key} for' f'Got invalid key value {key} for'
f' dict key at \'{fieldpath}\' on {cls};' f' dict key at \'{fieldpath}\''
f' on {cls.__name__};'
f' expected a value corresponding to' f' expected a value corresponding to'
f' a {keyanntype}.') from exc f' a {keyanntype}.') from exc
out[enumval] = self._value_from_input( out[enumval] = self._value_from_input(
@ -1182,7 +1207,8 @@ class _Inputter(Generic[T]):
except (ValueError, TypeError) as exc: except (ValueError, TypeError) as exc:
raise ValueError( raise ValueError(
f'Got invalid key value {key} for' f'Got invalid key value {key} for'
f' dict key at \'{fieldpath}\' on {cls};' f' dict key at \'{fieldpath}\''
f' on {cls.__name__};'
f' expected {keyanntype} value (though' f' expected {keyanntype} value (though'
f' in string form).') from exc f' in string form).') from exc
out[enumval] = self._value_from_input( out[enumval] = self._value_from_input(
@ -1208,7 +1234,7 @@ class _Inputter(Generic[T]):
# and then just grab them. # and then just grab them.
if len(childanntypes) == 0 or childanntypes[0] is typing.Any: if len(childanntypes) == 0 or childanntypes[0] is typing.Any:
for i, child in enumerate(value): for i, child in enumerate(value):
if not _is_valid_json(child): if not _is_valid_for_codec(child, self._codec):
raise TypeError(f'Item {i} of {fieldpath} contains' raise TypeError(f'Item {i} of {fieldpath} contains'
f' data type(s) not supported by json.') f' data type(s) not supported by json.')
return value if type(value) is seqtype else seqtype(value) return value if type(value) is seqtype else seqtype(value)
@ -1229,7 +1255,8 @@ class _Inputter(Generic[T]):
# a subclass with extended precision. # a subclass with extended precision.
if not isinstance(value, datetime.datetime): if not isinstance(value, datetime.datetime):
raise TypeError( raise TypeError(
f'Invalid input value for "{fieldpath}" on "{cls}";' f'Invalid input value for "{fieldpath}" on'
f' "{cls.__name__}";'
f' expected a datetime, got a {type(value).__name__}') f' expected a datetime, got a {type(value).__name__}')
_ensure_datetime_is_timezone_aware(value) _ensure_datetime_is_timezone_aware(value)
return value return value
@ -1239,11 +1266,11 @@ class _Inputter(Generic[T]):
# We expect a list of 7 ints. # We expect a list of 7 ints.
if type(value) is not list: if type(value) is not list:
raise TypeError( raise TypeError(
f'Invalid input value for "{fieldpath}" on "{cls}";' f'Invalid input value for "{fieldpath}" on "{cls.__name__}";'
f' expected a list, got a {type(value).__name__}') f' expected a list, got a {type(value).__name__}')
if len(value) != 7 or not all(isinstance(x, int) for x in value): if len(value) != 7 or not all(isinstance(x, int) for x in value):
raise TypeError( raise TypeError(
f'Invalid input value for "{fieldpath}" on "{cls}";' f'Invalid input value for "{fieldpath}" on "{cls.__name__}";'
f' expected a list of 7 ints.') f' expected a list of 7 ints.')
out = datetime.datetime( # type: ignore out = datetime.datetime( # type: ignore
*value, tzinfo=datetime.timezone.utc) *value, tzinfo=datetime.timezone.utc)
@ -1277,7 +1304,7 @@ class _Inputter(Generic[T]):
# 'Any' type children; make sure they are valid json values # 'Any' type children; make sure they are valid json values
# and then just grab them. # and then just grab them.
if childanntype is typing.Any: if childanntype is typing.Any:
if not _is_valid_json(childval): if not _is_valid_for_codec(childval, self._codec):
raise TypeError(f'Item {i} of {fieldpath} contains' raise TypeError(f'Item {i} of {fieldpath} contains'
f' data type(s) not supported by json.') f' data type(s) not supported by json.')
out.append(childval) out.append(childval)