diff --git a/.efrocachemap b/.efrocachemap
index 0507a2f8..620e11fc 100644
--- a/.efrocachemap
+++ b/.efrocachemap
@@ -3932,26 +3932,26 @@
"assets/build/windows/Win32/ucrtbased.dll": "https://files.ballistica.net/cache/ba1/b5/85/f8b6d0558ddb87267f34254b1450",
"assets/build/windows/Win32/vc_redist.x86.exe": "https://files.ballistica.net/cache/ba1/1c/e1/4a1a2eddda2f4aebd5f8b64ab08e",
"assets/build/windows/Win32/vcruntime140d.dll": "https://files.ballistica.net/cache/ba1/50/8d/bc2600ac9491f1b14d659709451f",
- "build/prefab/full/linux_arm64/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/2b/e1/628191b4c0b260036e0d7abc6ee3",
- "build/prefab/full/linux_arm64/release/ballisticacore": "https://files.ballistica.net/cache/ba1/83/51/0511414b7d61f636b55bb6e63b00",
- "build/prefab/full/linux_arm64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/83/a1/3046fc4c86fb31655da6b2d076cb",
- "build/prefab/full/linux_arm64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/ba/e7/d0c78aaf3c6982eee87471ff5637",
- "build/prefab/full/linux_x86_64/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/a3/43/5e2dd209bedf7f3c5550f416fb6e",
- "build/prefab/full/linux_x86_64/release/ballisticacore": "https://files.ballistica.net/cache/ba1/7b/e0/5c94fd10dfb247529548a76ddd75",
- "build/prefab/full/linux_x86_64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/13/54/1e3e4532fa176cca029fc448ac6e",
- "build/prefab/full/linux_x86_64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/d9/5b/d8a9b7d2cda3eaff208944273b08",
- "build/prefab/full/mac_arm64/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/3b/37/1055f167fbab28003f7129e5aee6",
- "build/prefab/full/mac_arm64/release/ballisticacore": "https://files.ballistica.net/cache/ba1/25/aa/6f6c4ef8ab7d39f46b8bf69fa18b",
- "build/prefab/full/mac_arm64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/f3/c4/619b03c16992b06d8162cdb96126",
- "build/prefab/full/mac_arm64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/2f/1b/6ec4f564ea77d42a30917c8434c9",
- "build/prefab/full/mac_x86_64/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/b5/73/194f8c35a467b775eebe24de90a2",
- "build/prefab/full/mac_x86_64/release/ballisticacore": "https://files.ballistica.net/cache/ba1/b0/a7/7848042fbcd5764af31020e53ae1",
- "build/prefab/full/mac_x86_64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/28/a0/d2d53829a074e5c06cf5785af1cd",
- "build/prefab/full/mac_x86_64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/4d/53/3027bbbab3c19fa2b8d0fa718d96",
- "build/prefab/full/windows_x86/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/f9/c7/b04b34fcdc0e0e0f9976b5639c0c",
- "build/prefab/full/windows_x86/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/87/a9/e295e82a1f3b423fb2f8f2eb462f",
- "build/prefab/full/windows_x86_server/debug/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/1c/bd/4afc4c92dcaf095928f66d5c61d3",
- "build/prefab/full/windows_x86_server/release/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/ca/56/0923f65bb7c80211f614c4816ab5",
+ "build/prefab/full/linux_arm64/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/93/85/ceb5bcda0a0a8b9cde91e599f505",
+ "build/prefab/full/linux_arm64/release/ballisticacore": "https://files.ballistica.net/cache/ba1/76/38/49d10532844bfe7df52b53216d2b",
+ "build/prefab/full/linux_arm64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/28/8a/26ac9d06cc08010e5ef47fe1e14b",
+ "build/prefab/full/linux_arm64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/b5/71/06235b13926864357c4b6cff841c",
+ "build/prefab/full/linux_x86_64/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/2e/c5/61ca27f1d369d2e11025c9b41c13",
+ "build/prefab/full/linux_x86_64/release/ballisticacore": "https://files.ballistica.net/cache/ba1/d9/58/78661f5aecd0da126b10201d0c5e",
+ "build/prefab/full/linux_x86_64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/36/be/d706c897e778fa1b0ba439eff8c1",
+ "build/prefab/full/linux_x86_64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/eb/15/84f0a16a78fee48970391300b957",
+ "build/prefab/full/mac_arm64/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/60/eb/4f65457cf792f721bbad617d9ad0",
+ "build/prefab/full/mac_arm64/release/ballisticacore": "https://files.ballistica.net/cache/ba1/71/f1/4f36fb59372af667d2fcfc16c6d8",
+ "build/prefab/full/mac_arm64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/7f/e6/13d77d7939fa00e57f16ed440f92",
+ "build/prefab/full/mac_arm64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/44/2d/1132879368551cc7bd7fb407f0fd",
+ "build/prefab/full/mac_x86_64/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/77/6e/d9920eeaa9b9adba4cb3f5e655f0",
+ "build/prefab/full/mac_x86_64/release/ballisticacore": "https://files.ballistica.net/cache/ba1/b9/02/ed3c91a260a476a66baab752e75c",
+ "build/prefab/full/mac_x86_64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/56/ca/d159a3a9dcb1f266413578651e16",
+ "build/prefab/full/mac_x86_64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/c9/5e/9cc46cf58951c133917a04f04cd0",
+ "build/prefab/full/windows_x86/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/90/84/390b6916bbf2b6710729a923ab7c",
+ "build/prefab/full/windows_x86/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/3e/f8/36ae7dc2ac79f3838999f66e1c87",
+ "build/prefab/full/windows_x86_server/debug/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/fe/a1/b217231f196dc175663d7b70b4a9",
+ "build/prefab/full/windows_x86_server/release/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/95/7a/a961123e1b9bcd78a563759fd2cf",
"build/prefab/lib/linux_arm64/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/54/e0/8c89d241ca37ae32c9b46ea71456",
"build/prefab/lib/linux_arm64/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/e0/05/94ac67cbf2e665eeecfc77246caf",
"build/prefab/lib/linux_arm64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/0f/e1/be897ef0e73b0e52c59f3396e0cd",
@@ -3960,12 +3960,12 @@
"build/prefab/lib/linux_x86_64/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/1f/b3/80375870a9ab83bbb63b50b03a73",
"build/prefab/lib/linux_x86_64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/5c/2e/a1b1f2126b3150c6f11e799c3805",
"build/prefab/lib/linux_x86_64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/b6/6b/37174b3a0d72a05f1bb3d904b78b",
- "build/prefab/lib/mac_arm64/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/43/b7/8f87aa950891250923c7cb617d32",
- "build/prefab/lib/mac_arm64/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/1a/e9/a0c9c846c00b5ed9645ddc70e44a",
- "build/prefab/lib/mac_arm64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/8a/e5/1f93415fb55842479f5ce0d375af",
- "build/prefab/lib/mac_arm64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/e7/36/2475ad00f41cf75d63c786f3db39",
- "build/prefab/lib/mac_x86_64/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/6c/52/ff241eeae0d43f433f66a8b2dcbb",
- "build/prefab/lib/mac_x86_64/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/40/a3/a2c2c741e551c40d0700888737a7",
- "build/prefab/lib/mac_x86_64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/a1/ef/fa977c1c8a05781f6af8321b31f2",
- "build/prefab/lib/mac_x86_64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/0d/15/336d51f7894aeedba54f5aabb04a"
+ "build/prefab/lib/mac_arm64/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/c4/c3/de4c713a14762b156cbc9f1dbacc",
+ "build/prefab/lib/mac_arm64/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/e8/a5/7341744d46bf83683f6d9113019a",
+ "build/prefab/lib/mac_arm64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/3b/c7/76236792620e2f0a7a5551b32a45",
+ "build/prefab/lib/mac_arm64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/86/6d/2f7220ea5ab89b23ffacc7497694",
+ "build/prefab/lib/mac_x86_64/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/9f/46/d0cd8a310edfa84f124d11a295d7",
+ "build/prefab/lib/mac_x86_64/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/d7/7d/5196bbacb6efbfd4fe70cc042553",
+ "build/prefab/lib/mac_x86_64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/73/63/38d1af71edf49e29f802533058a0",
+ "build/prefab/lib/mac_x86_64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/d9/08/502699c51e7a36c1b60acec6be36"
}
\ No newline at end of file
diff --git a/.idea/dictionaries/ericf.xml b/.idea/dictionaries/ericf.xml
index 08230e08..61860599 100644
--- a/.idea/dictionaries/ericf.xml
+++ b/.idea/dictionaries/ericf.xml
@@ -1088,6 +1088,7 @@
intp
introspectable
intstr
+ ioattrs
iobj
iometa
ioprep
diff --git a/ballisticacore-cmake/.idea/dictionaries/ericf.xml b/ballisticacore-cmake/.idea/dictionaries/ericf.xml
index 613b7565..ad984bf8 100644
--- a/ballisticacore-cmake/.idea/dictionaries/ericf.xml
+++ b/ballisticacore-cmake/.idea/dictionaries/ericf.xml
@@ -488,6 +488,7 @@
interuptions
intstr
invote
+ ioattrs
iobj
iometa
ioprep
diff --git a/docs/ba_module.md b/docs/ba_module.md
index 8f3a4fee..51641da7 100644
--- a/docs/ba_module.md
+++ b/docs/ba_module.md
@@ -1,5 +1,5 @@
-last updated on 2021-05-24 for Ballistica version 1.6.3 build 20367
+last updated on 2021-05-25 for Ballistica version 1.6.4 build 20369
This page documents the Python classes and functions in the 'ba' module,
which are the ones most relevant to modding in Ballistica. If you come across something you feel should be included here or could be better explained, please let me know. Happy modding!
diff --git a/src/ballistica/ballistica.cc b/src/ballistica/ballistica.cc
index fa8238c9..e7459ce9 100644
--- a/src/ballistica/ballistica.cc
+++ b/src/ballistica/ballistica.cc
@@ -21,8 +21,8 @@
namespace ballistica {
// These are set automatically via script; don't change here.
-const int kAppBuildNumber = 20368;
-const char* kAppVersion = "1.6.3";
+const int kAppBuildNumber = 20369;
+const char* kAppVersion = "1.6.4";
// Our standalone globals.
// These are separated out for easy access.
diff --git a/tests/test_efro/test_dataclassio.py b/tests/test_efro/test_dataclassio.py
index 030f771f..f2e00cbd 100644
--- a/tests/test_efro/test_dataclassio.py
+++ b/tests/test_efro/test_dataclassio.py
@@ -15,7 +15,7 @@ import pytest
from efro.util import utc_now
from efro.dataclassio import (dataclass_validate, dataclass_from_dict,
- dataclass_to_dict, ioprepped, IOMeta)
+ dataclass_to_dict, ioprepped, IOAttrs, Codec)
if TYPE_CHECKING:
pass
@@ -461,13 +461,13 @@ def test_extra_data() -> None:
assert 'nonexistent' not in out
-def test_meta() -> None:
- """Testing iometa annotations."""
+def test_ioattrs() -> None:
+ """Testing ioattrs annotations."""
@ioprepped
@dataclass
class _TestClass:
- dval: Annotated[Dict, IOMeta('d')]
+ dval: Annotated[Dict, IOAttrs('d')]
obj = _TestClass(dval={'foo': 'bar'})
@@ -481,14 +481,14 @@ def test_meta() -> None:
@ioprepped
@dataclass
class _TestClass2:
- dval: Annotated[Dict, IOMeta('d', store_default=False)]
+ dval: Annotated[Dict, IOAttrs('d', store_default=False)]
@ioprepped
@dataclass
class _TestClass3:
- dval: Annotated[Dict, IOMeta('d', store_default=False)] = field(
+ dval: Annotated[Dict, IOAttrs('d', store_default=False)] = field(
default_factory=dict)
- ival: Annotated[int, IOMeta('i', store_default=False)] = 123
+ ival: Annotated[int, IOAttrs('i', store_default=False)] = 123
# Both attrs are default; should get stripped out.
obj3 = _TestClass3()
@@ -513,6 +513,51 @@ def test_meta() -> None:
assert obj3.ival == 125
+def test_codecs() -> None:
+ """Test differences with codecs."""
+
+ @ioprepped
+ @dataclass
+ class _TestClass:
+ bval: bytes
+
+ # bytes to/from JSON (goes through base64)
+ obj = _TestClass(bval=b'foo')
+ out = dataclass_to_dict(obj, codec=Codec.JSON)
+ assert isinstance(out['bval'], str) and out['bval'] == 'Zm9v'
+ obj = dataclass_from_dict(_TestClass, out, codec=Codec.JSON)
+ assert obj.bval == b'foo'
+
+ # bytes to/from FIRESTORE (passed as-is)
+ obj = _TestClass(bval=b'foo')
+ out = dataclass_to_dict(obj, codec=Codec.FIRESTORE)
+ assert isinstance(out['bval'], bytes) and out['bval'] == b'foo'
+ obj = dataclass_from_dict(_TestClass, out, codec=Codec.FIRESTORE)
+ assert obj.bval == b'foo'
+
+ now = utc_now()
+
+ @ioprepped
+ @dataclass
+ class _TestClass2:
+ dval: datetime.datetime
+
+ # datetime to/from JSON (turns into a list of values)
+ obj2 = _TestClass2(dval=now)
+ out = dataclass_to_dict(obj2, codec=Codec.JSON)
+ assert (isinstance(out['dval'], list) and len(out['dval']) == 7
+ and all(isinstance(val, int) for val in out['dval']))
+ obj2 = dataclass_from_dict(_TestClass2, out, codec=Codec.JSON)
+ assert obj2.dval == now
+
+ # datetime to/from FIRESTORE (passed through as-is)
+ obj2 = _TestClass2(dval=now)
+ out = dataclass_to_dict(obj2, codec=Codec.FIRESTORE)
+ assert isinstance(out['dval'], datetime.datetime)
+ obj2 = dataclass_from_dict(_TestClass2, out, codec=Codec.FIRESTORE)
+ assert obj2.dval == now
+
+
def test_dict() -> None:
"""Test various dict related bits."""
diff --git a/tools/efro/dataclassio.py b/tools/efro/dataclassio.py
index 3d6ad3ae..320f49ad 100644
--- a/tools/efro/dataclassio.py
+++ b/tools/efro/dataclassio.py
@@ -57,15 +57,21 @@ PREP_ATTR = '_DCIOPREP'
EXTRA_ATTRS_ATTR = '_DCIOEXATTRS'
-class IOMeta:
- """Metadata for specifying io behavior."""
+class Codec(Enum):
+ """Influences format used for input/output."""
+ JSON = 'json'
+ FIRESTORE = 'firestore'
+
+
+class IOAttrs:
+ """For specifying io behavior in annotations."""
def __init__(self, storagename: str = None, store_default: bool = True):
self.storagename = storagename
self.store_default = store_default
def validate_for_field(self, cls: Type, field: dataclasses.Field) -> None:
- """Ensure the IOMeta instance is ok to use with the provided field."""
+ """Ensure the IOAttrs instance is ok to use with the provided field."""
# Turning off store_default requires the field to have either
# a default_factory or a default
@@ -78,7 +84,9 @@ class IOMeta:
f' store_default=False cannot be set for it.')
-def dataclass_to_dict(obj: Any, coerce_to_float: bool = True) -> dict:
+def dataclass_to_dict(obj: Any,
+ codec: Codec = Codec.JSON,
+ coerce_to_float: bool = True) -> dict:
"""Given a dataclass object, return a json-friendly dict.
All values will be checked to ensure they match the types specified
@@ -95,7 +103,10 @@ def dataclass_to_dict(obj: Any, coerce_to_float: bool = True) -> dict:
will be triggered.
"""
- out = _Outputter(obj, create=True, coerce_to_float=coerce_to_float).run()
+ out = _Outputter(obj,
+ create=True,
+ codec=codec,
+ coerce_to_float=coerce_to_float).run()
assert isinstance(out, dict)
return out
@@ -107,21 +118,24 @@ def dataclass_to_json(obj: Any, coerce_to_float: bool = True) -> str:
"""
import json
return json.dumps(
- dataclass_to_dict(obj=obj, coerce_to_float=coerce_to_float),
+ dataclass_to_dict(obj=obj,
+ coerce_to_float=coerce_to_float,
+ codec=Codec.JSON),
separators=(',', ':'),
)
def dataclass_from_dict(cls: Type[T],
values: dict,
+ codec: Codec = Codec.JSON,
coerce_to_float: bool = True,
allow_unknown_attrs: bool = True,
discard_unknown_attrs: bool = False) -> T:
"""Given a dict, return a dataclass of a given type.
- The dict must be in the json-friendly format as emitted from
- dataclass_to_dict. This means that sequence values such as tuples or
- sets should be passed as lists, enums should be passed as their
+ The dict must be formatted to match the specified codec (generally
+ json-friendly object types). This means that sequence values such as
+ tuples or sets should be passed as lists, enums should be passed as their
associated values, nested dataclasses should be passed as dicts, etc.
All values are checked to ensure their types/values are valid.
@@ -141,6 +155,7 @@ def dataclass_from_dict(cls: Type[T],
case they will simply be discarded.
"""
return _Inputter(cls,
+ codec=codec,
coerce_to_float=coerce_to_float,
allow_unknown_attrs=allow_unknown_attrs,
discard_unknown_attrs=discard_unknown_attrs).run(values)
@@ -163,12 +178,15 @@ def dataclass_from_json(cls: Type[T],
discard_unknown_attrs=discard_unknown_attrs)
-def dataclass_validate(obj: Any, coerce_to_float: bool = True) -> None:
+def dataclass_validate(obj: Any,
+ coerce_to_float: bool = True,
+ codec: Codec = Codec.JSON) -> None:
"""Ensure that values in a dataclass instance are the correct types."""
# Simply run an output pass but tell it not to generate data;
# only run validation.
- _Outputter(obj, create=False, coerce_to_float=coerce_to_float).run()
+ _Outputter(obj, create=False, codec=codec,
+ coerce_to_float=coerce_to_float).run()
def ioprep(cls: Type) -> None:
@@ -186,10 +204,7 @@ def ioprep(cls: Type) -> None:
Prepping a dataclass involves evaluating its type annotations, which,
as of PEP 563, are stored simply as strings. This evaluation is done
in the module namespace containing the class, so all referenced types
- must be defined at that level. The exception is Typing types (Optional,
- Union, etc.) which are often defined under an 'if TYPE_CHECKING'
- conditional and thus not available at runtime, so are explicitly made
- available during annotation evaluation.
+ must be defined at that level.
"""
PrepSession(explicit=True).prep_dataclass(cls, recursion_level=0)
@@ -200,7 +215,8 @@ def ioprepped(cls: Type[T]) -> Type[T]:
Note that in some cases it may not be possible to prep a dataclass
immediately (such as when its type annotations refer to forward-declared
types). In these cases, dataclass_prep() should be explicitly called for
- the class once it is safe to do so.
+ the class as soon as possible; ideally at module import time to expose any
+ errors as early as possible in execution.
"""
ioprep(cls)
return cls
@@ -278,14 +294,14 @@ class PrepSession:
# types and prepping any contained dataclass types.
for attrname, anntype in resolved_annotations.items():
- anntype, iometa = _parse_annotated(anntype)
+ anntype, ioattrs = _parse_annotated(anntype)
- # If we found attached IOMeta data, make sure it contains
+ # If we found attached IOAttrs data, make sure it contains
# valid values for the field it is attached to.
- if iometa is not None:
- iometa.validate_for_field(cls, fields_by_name[attrname])
- if iometa.storagename is not None:
- storage_names_to_attr_names[iometa.storagename] = attrname
+ if ioattrs is not None:
+ ioattrs.validate_for_field(cls, fields_by_name[attrname])
+ if ioattrs.storagename is not None:
+ storage_names_to_attr_names[ioattrs.storagename] = attrname
self.prep_type(cls,
attrname,
@@ -325,15 +341,10 @@ class PrepSession:
# Everything below this point assumes the annotation type resolves
# to a concrete type.
if not isinstance(origin, type):
- print('ORIGIN IS', origin, type(origin))
raise TypeError(
f'Unsupported type found for \'{attrname}\' on {cls}:'
f' {anntype}')
- # extras = get_args(anntype)
- # if extras:
- # print('FOUND EXTRAS FOR', anntype)
-
if origin in SIMPLE_TYPES:
return
@@ -417,6 +428,9 @@ class PrepSession:
self.prep_dataclass(origin, recursion_level=recursion_level + 1)
return
+ if origin is bytes:
+ return
+
raise TypeError(f"Attr '{attrname}' on {cls} contains type '{anntype}'"
f' which is not supported by dataclassio.')
@@ -508,9 +522,11 @@ def _get_origin(anntype: Any) -> Any:
class _Outputter:
"""Validates or exports data contained in a dataclass instance."""
- def __init__(self, obj: Any, create: bool, coerce_to_float: bool) -> None:
+ def __init__(self, obj: Any, create: bool, codec: Codec,
+ coerce_to_float: bool) -> None:
self._obj = obj
self._create = create
+ self._codec = codec
self._coerce_to_float = coerce_to_float
def run(self) -> Any:
@@ -533,11 +549,11 @@ class _Outputter:
anntype = prep.annotations[fieldname]
value = getattr(obj, fieldname)
- anntype, iometa = _parse_annotated(anntype)
+ anntype, ioattrs = _parse_annotated(anntype)
# If we're not storing default values for this fella,
# we can skip all output processing if we've got a default value.
- if iometa is not None and not iometa.store_default:
+ if ioattrs is not None and not ioattrs.store_default:
default_factory: Any = field.default_factory # type: ignore
if default_factory is not dataclasses.MISSING:
if default_factory() == value:
@@ -556,8 +572,8 @@ class _Outputter:
if self._create:
assert out is not None
storagename = (fieldname if
- (iometa is None or iometa.storagename is None)
- else iometa.storagename)
+ (ioattrs is None or ioattrs.storagename is None)
+ else ioattrs.storagename)
out[storagename] = outvalue
# If there's extra-attrs stored on us, check/include them.
@@ -712,19 +728,37 @@ class _Outputter:
if not isinstance(value, origin):
raise TypeError(f'Expected a {origin} for {fieldpath};'
f' found a {type(value)}.')
- # We only support timezone-aware utc times.
- if (value.tzinfo is not datetime.timezone.utc
- and (_pytz_utc is None or value.tzinfo is not _pytz_utc)):
- raise ValueError(
- 'datetime values must have timezone set as timezone.utc')
+ _ensure_datetime_is_timezone_aware(value)
+ if self._codec is Codec.FIRESTORE:
+ return value
+ assert self._codec is Codec.JSON
return [
value.year, value.month, value.day, value.hour, value.minute,
value.second, value.microsecond
] if self._create else None
+ if origin is bytes:
+ return self._process_bytes(cls, fieldpath, value)
+
raise TypeError(
f"Field '{fieldpath}' of type '{anntype}' is unsupported here.")
+ def _process_bytes(self, cls: Type, fieldpath: str, value: bytes) -> Any:
+ import base64
+ if not isinstance(value, bytes):
+ raise TypeError(f'Expected bytes for {fieldpath} on {cls};'
+ f' found a {type(value)}.')
+
+ if not self._create:
+ return None
+
+ # In JSON we convert to base64, but firestore directly supports bytes.
+ if self._codec is Codec.JSON:
+ return base64.b64encode(value).decode()
+
+ assert self._codec is Codec.FIRESTORE
+ return value
+
def _process_dict(self, cls: Type, fieldpath: str, anntype: Any,
value: dict) -> Any:
# pylint: disable=too-many-branches
@@ -792,10 +826,12 @@ class _Inputter(Generic[T]):
def __init__(self,
cls: Type[T],
+ codec: Codec,
coerce_to_float: bool,
allow_unknown_attrs: bool = True,
discard_unknown_attrs: bool = False):
self._cls = cls
+ self._codec = codec
self._coerce_to_float = coerce_to_float
self._allow_unknown_attrs = allow_unknown_attrs
self._discard_unknown_attrs = discard_unknown_attrs
@@ -871,9 +907,32 @@ class _Inputter(Generic[T]):
if issubclass(origin, datetime.datetime):
return self._datetime_from_input(cls, fieldpath, value)
+ if origin is bytes:
+ return self._bytes_from_input(origin, fieldpath, value)
+
raise TypeError(
f"Field '{fieldpath}' of type '{anntype}' is unsupported here.")
+ def _bytes_from_input(self, cls: Type, fieldpath: str,
+ value: Any) -> bytes:
+ """Given input data, returns bytes."""
+ import base64
+
+ # For firestore, bytes are passed as-is. Otherwise they're encoded
+ # as base64.
+ if self._codec is Codec.FIRESTORE:
+ if not isinstance(value, bytes):
+ raise TypeError(f'Expected a bytes object for {fieldpath}'
+ f' on {cls}; got a {type(value)}.')
+
+ return value
+
+ assert self._codec is Codec.JSON
+ if not isinstance(value, str):
+ raise TypeError(f'Expected a string object for {fieldpath}'
+ f' on {cls}; got a {type(value)}.')
+ return base64.b64decode(value)
+
def _dataclass_from_input(self, cls: Type, fieldpath: str,
values: dict) -> Any:
"""Given a dict, instantiates a dataclass of the given type.
@@ -885,7 +944,8 @@ class _Inputter(Generic[T]):
"""
# pylint: disable=too-many-locals
if not isinstance(values, dict):
- raise TypeError("Expected a dict for 'values' arg.")
+ raise TypeError(f'Expected a dict for {fieldpath} on {cls};'
+ f' got a {type(values)}.')
prep = PrepSession(explicit=False).prep_dataclass(cls,
recursion_level=0)
@@ -920,7 +980,7 @@ class _Inputter(Generic[T]):
else:
fieldname = field.name
anntype = prep.annotations[fieldname]
- anntype, _iometa = _parse_annotated(anntype)
+ anntype, _ioattrs = _parse_annotated(anntype)
subfieldpath = (f'{fieldpath}.{fieldname}'
if fieldpath else fieldname)
@@ -1058,6 +1118,19 @@ class _Inputter(Generic[T]):
def _datetime_from_input(self, cls: Type, fieldpath: str,
value: Any) -> Any:
+ # For firestore we expect a datetime object.
+ if self._codec is Codec.FIRESTORE:
+ # Don't compare exact type here, as firestore can give us
+ # a subclass with extended precision.
+ if not isinstance(value, datetime.datetime):
+ raise TypeError(
+ f'Invalid input value for "{fieldpath}" on "{cls}";'
+ f' expected a datetime, got a {type(value).__name__}')
+ _ensure_datetime_is_timezone_aware(value)
+ return value
+
+ assert self._codec is Codec.JSON
+
# We expect a list of 7 ints.
if type(value) is not list:
raise TypeError(
@@ -1109,20 +1182,28 @@ class _Inputter(Generic[T]):
return tuple(out)
-def _parse_annotated(anntype: Any) -> Tuple[Any, Optional[IOMeta]]:
- """Parse Annotated() constructs, returning annotated type & IOMeta data."""
+def _ensure_datetime_is_timezone_aware(value: datetime.datetime) -> None:
+ # We only support timezone-aware utc times.
+ if (value.tzinfo is not datetime.timezone.utc
+ and (_pytz_utc is None or value.tzinfo is not _pytz_utc)):
+ raise ValueError(
+ 'datetime values must have timezone set as timezone.utc')
+
+
+def _parse_annotated(anntype: Any) -> Tuple[Any, Optional[IOAttrs]]:
+ """Parse Annotated() constructs, returning annotated type & IOAttrs."""
# If we get an Annotated[foo, bar, eep] we take
- # foo as the actual type and we look for IOMeta instances in
+ # foo as the actual type and we look for IOAttrs instances in
# bar/eep to affect our behavior.
- iometa: Optional[IOMeta] = None
+ ioattrs: Optional[IOAttrs] = None
if isinstance(anntype, _AnnotatedAlias):
annargs = get_args(anntype)
for annarg in annargs[1:]:
- if isinstance(annarg, IOMeta):
- if iometa is not None:
+ if isinstance(annarg, IOAttrs):
+ if ioattrs is not None:
raise RuntimeError(
- 'Multiple IOMeta instances found for a'
+ 'Multiple IOAttrs instances found for a'
' single annotation; this is not supported.')
- iometa = annarg
+ ioattrs = annarg
anntype = annargs[0]
- return anntype, iometa
+ return anntype, ioattrs