diff --git a/.idea/dictionaries/ericf.xml b/.idea/dictionaries/ericf.xml index 12147f94..f35449fc 100644 --- a/.idea/dictionaries/ericf.xml +++ b/.idea/dictionaries/ericf.xml @@ -83,7 +83,12 @@ assetbundle assetcache assetdata + assetpack assetpackage + assetpackput + assetpackputfinish + assetpackputmanifest + assetpackputupload assetpath assettype assettypestr @@ -526,6 +531,7 @@ fhdr fieldattr fieldtypes + filebytes filecmp filehash fileinfo @@ -534,6 +540,7 @@ filenames filepath fileselector + filesize filestates filterlines filterpath @@ -1129,6 +1136,7 @@ packagedir packagedirs packagename + packageversion painttxtattr palmos pandoc @@ -1777,6 +1785,7 @@ vsync vsyncs vval + waaah wanttype wasdead weakref diff --git a/assets/src/data/scripts/bafoundation/entity/_entity.py b/assets/src/data/scripts/bafoundation/entity/_entity.py index 7af497fb..e7f54698 100644 --- a/assets/src/data/scripts/bafoundation/entity/_entity.py +++ b/assets/src/data/scripts/bafoundation/entity/_entity.py @@ -45,7 +45,7 @@ class EntityMixin: def __init__(self, d_data: Dict[str, Any] = None, - error: bool = False) -> None: + error: bool = True) -> None: super().__init__() if not isinstance(self, CompoundValue): raise RuntimeError('EntityMixin class must be combined' @@ -60,7 +60,7 @@ class EntityMixin: """Resets data to default.""" self.set_data({}, error=True) - def set_data(self, data: Dict, error: bool = False) -> None: + def set_data(self, data: Dict, error: bool = True) -> None: """Set the data for this entity and apply all value filters to it. Note that it is more efficient to pass data to an Entity's constructor @@ -159,7 +159,7 @@ class EntityMixin: """ return json.loads(s, cls=ExtendedJSONDecoder) - def load_from_json_str(self, s: str, error: bool = False) -> None: + def load_from_json_str(self, s: str, error: bool = True) -> None: """Set the entity's data in-place from a json string. The 'error' argument determines whether Exceptions will be raised @@ -171,7 +171,7 @@ class EntityMixin: self.set_data(data, error=error) @classmethod - def from_json_str(cls: Type[T], s: str, error: bool = False) -> T: + def from_json_str(cls: Type[T], s: str, error: bool = True) -> T: """Instantiate a new instance with provided json string. The 'error' argument determines whether exceptions will be raised diff --git a/assets/src/data/scripts/bafoundation/entity/_field.py b/assets/src/data/scripts/bafoundation/entity/_field.py index 3b0236b5..2fbfa7ac 100644 --- a/assets/src/data/scripts/bafoundation/entity/_field.py +++ b/assets/src/data/scripts/bafoundation/entity/_field.py @@ -34,7 +34,6 @@ from bafoundation.entity._support import (BaseField, BoundCompoundValue, if TYPE_CHECKING: from typing import Dict, Type, List, Any from bafoundation.entity._value import TypedValue, CompoundValue - from bafoundation.entity._support import FieldInspector T = TypeVar('T') TK = TypeVar('TK') @@ -46,7 +45,7 @@ class Field(BaseField, Generic[T]): def __init__(self, d_key: str, - value: 'TypedValue[T]', + value: TypedValue[T], store_default: bool = True) -> None: super().__init__(d_key) self.d_value = value @@ -73,9 +72,17 @@ class Field(BaseField, Generic[T]): # a type instead of an instance, but we don't reflect that here yet # (would need to write a mypy plugin so sub-field access works first) + @overload + def __get__(self, obj: None, cls: Any = None) -> Field[T]: + ... + + @overload def __get__(self, obj: Any, cls: Any = None) -> T: ... + def __get__(self, obj: Any, cls: Any = None) -> Any: + ... + def __set__(self, obj: Any, value: T) -> None: ... @@ -164,7 +171,7 @@ class ListField(BaseField, Generic[T]): def __init__(self, d_key: str, - value: 'TypedValue[T]', + value: TypedValue[T], store_default: bool = True) -> None: super().__init__(d_key) self.d_value = value @@ -174,9 +181,14 @@ class ListField(BaseField, Generic[T]): return [] def filter_input(self, data: Any, error: bool) -> Any: + + # If we were passed a BoundListField, operate on its raw values + if isinstance(data, BoundListField): + data = data.d_data + if not isinstance(data, list): if error: - raise TypeError('list value expected') + raise TypeError(f'list value expected; got {type(data)}') logging.error('Ignoring non-list data for %s: %s', self, data) data = [] for i, entry in enumerate(data): @@ -193,8 +205,9 @@ class ListField(BaseField, Generic[T]): if TYPE_CHECKING: + # Access via type gives our field; via an instance gives a bound field. @overload - def __get__(self, obj: None, cls: Any = None) -> FieldInspector: + def __get__(self, obj: None, cls: Any = None) -> ListField[T]: ... @overload @@ -204,9 +217,18 @@ class ListField(BaseField, Generic[T]): def __get__(self, obj: Any, cls: Any = None) -> Any: ... + # Allow setting via a raw value list or a bound list field + @overload def __set__(self, obj: Any, value: List[T]) -> None: ... + @overload + def __set__(self, obj: Any, value: BoundListField[T]) -> None: + ... + + def __set__(self, obj: Any, value: Any) -> None: + ... + def get_with_data(self, data: Any) -> Any: return BoundListField(self, data[self.d_key]) @@ -217,7 +239,7 @@ class DictField(BaseField, Generic[TK, T]): def __init__(self, d_key: str, keytype: Type[TK], - field: 'TypedValue[T]', + field: TypedValue[T], store_default: bool = True) -> None: super().__init__(d_key) self.d_value = field @@ -229,6 +251,11 @@ class DictField(BaseField, Generic[TK, T]): # noinspection DuplicatedCode def filter_input(self, data: Any, error: bool) -> Any: + + # If we were passed a BoundDictField, operate on its raw values + if isinstance(data, BoundDictField): + data = data.d_data + if not isinstance(data, dict): if error: raise TypeError('dict value expected') @@ -252,6 +279,8 @@ class DictField(BaseField, Generic[TK, T]): if TYPE_CHECKING: + # Return our field if accessed via type and bound-dict-field + # if via instance. @overload def __get__(self, obj: None, cls: Any = None) -> DictField[TK, T]: ... @@ -263,9 +292,18 @@ class DictField(BaseField, Generic[TK, T]): def __get__(self, obj: Any, cls: Any = None) -> Any: ... + # Allow setting via matching dict values or BoundDictFields + @overload def __set__(self, obj: Any, value: Dict[TK, T]) -> None: ... + @overload + def __set__(self, obj: Any, value: BoundDictField[TK, T]) -> None: + ... + + def __set__(self, obj: Any, value: Any) -> None: + ... + def get_with_data(self, data: Any) -> Any: return BoundDictField(self._keytype, self, data[self.d_key]) @@ -290,6 +328,7 @@ class CompoundListField(BaseField, Generic[TC]): self._store_default = store_default def filter_input(self, data: Any, error: bool) -> list: + if not isinstance(data, list): if error: raise TypeError('list value expected') @@ -333,18 +372,34 @@ class CompoundListField(BaseField, Generic[TC]): # Note: # When setting the list, we tell the type-checker that we accept # a raw list of CompoundValue objects, but at runtime we actually - # deal with BoundCompoundValue objects (see note in + # always deal with BoundCompoundValue objects (see note in # BoundCompoundListField) + @overload def __set__(self, obj: Any, value: List[TC]) -> None: ... + @overload + def __set__(self, obj: Any, value: BoundCompoundListField[TC]) -> None: + ... + + def __set__(self, obj: Any, value: Any) -> None: + ... + def get_with_data(self, data: Any) -> Any: assert self.d_key in data return BoundCompoundListField(self, data[self.d_key]) def set_with_data(self, data: Any, value: Any, error: bool) -> Any: + + # If we were passed a BoundCompoundListField, + # simply convert it to a list of BoundCompoundValue objects which + # is what we work with natively here. + if isinstance(value, BoundCompoundListField): + value = list(value) + if not isinstance(value, list): - raise TypeError('CompoundListField expected list value on set.') + raise TypeError(f'CompoundListField expected list value on set;' + f' got {type(value)}.') # Allow assigning only from a sequence of our existing children. # (could look into expanding this to other children if we can @@ -355,7 +410,8 @@ class CompoundListField(BaseField, Generic[TC]): or not all(i.d_value is self.d_value for i in value)): raise ValueError('CompoundListField assignment must be a ' 'list containing only its existing children.') - data[self.d_key] = [i.d_data for i in value] + data[self.d_key] = self.filter_input([i.d_data for i in value], + error=error) class CompoundDictField(BaseField, Generic[TK, TC]): diff --git a/assets/src/data/scripts/bafoundation/entity/_support.py b/assets/src/data/scripts/bafoundation/entity/_support.py index 6f0f31fe..8c1b56fa 100644 --- a/assets/src/data/scripts/bafoundation/entity/_support.py +++ b/assets/src/data/scripts/bafoundation/entity/_support.py @@ -34,9 +34,9 @@ if TYPE_CHECKING: CompoundDictField) T = TypeVar('T') -TK = TypeVar('TK') -TC = TypeVar('TC', bound='CompoundValue') -TBL = TypeVar('TBL', bound='BoundCompoundListField') +TKey = TypeVar('TKey') +TCompound = TypeVar('TCompound', bound='CompoundValue') +TBoundList = TypeVar('TBoundList', bound='BoundCompoundListField') class BoundCompoundValue: @@ -49,12 +49,13 @@ class BoundCompoundValue: Dict[str, Any]]): self.d_value: CompoundValue self.d_data: Union[List[Any], Dict[str, Any]] - # need to use base setters to avoid triggering our own overrides + + # Need to use base setters to avoid triggering our own overrides. object.__setattr__(self, 'd_value', value) object.__setattr__(self, 'd_data', d_data) def __eq__(self, other: Any) -> Any: - # allow comparing to compound and bound-compound objects + # Allow comparing to compound and bound-compound objects. from bafoundation.entity.util import compound_eq return compound_eq(self, other) @@ -68,7 +69,7 @@ class BoundCompoundValue: raise AttributeError def __setattr__(self, name: str, value: Any) -> None: - # same deal as __getattr__ basically + # Same deal as __getattr__ basically. field = getattr(type(object.__getattribute__(self, 'd_value')), name, None) if isinstance(field, BaseField): @@ -81,10 +82,12 @@ class BoundCompoundValue: value = object.__getattribute__(self, 'd_value') data = object.__getattribute__(self, 'd_data') assert isinstance(data, dict) + # Need to clear our dict in-place since we have no # access to our parent which we'd need to assign an empty one. data.clear() - # now fill in default data + + # Now fill in default data. value.apply_fields_to_data(data, error=True) def __repr__(self) -> str: @@ -157,7 +160,7 @@ class BoundListField(Generic[T]): self._i = 0 def __eq__(self, other: Any) -> Any: - # just convert us into a regular list and run a compare with that + # Just convert us into a regular list and run a compare with that. flattened = [ self.d_field.d_value.filter_output(value) for value in self.d_data ] @@ -211,18 +214,18 @@ class BoundListField(Generic[T]): self.d_data[key] = self.d_field.d_value.filter_input(value, error=True) -class BoundDictField(Generic[TK, T]): +class BoundDictField(Generic[TKey, T]): """DictField bound to its data; used for accessing its values.""" - def __init__(self, keytype: Type[TK], field: DictField[TK, T], - d_data: Dict[TK, T]): + def __init__(self, keytype: Type[TKey], field: DictField[TKey, T], + d_data: Dict[TKey, T]): self._keytype = keytype self.d_field = field assert isinstance(d_data, dict) self.d_data = d_data def __eq__(self, other: Any) -> Any: - # just convert us into a regular dict and run a compare with that + # Just convert us into a regular dict and run a compare with that. flattened = { key: self.d_field.d_value.filter_output(value) for key, value in self.d_data.items() @@ -237,7 +240,7 @@ class BoundDictField(Generic[TK, T]): def __len__(self) -> int: return len(self.d_data) - def __getitem__(self, key: TK) -> T: + def __getitem__(self, key: TKey) -> T: if not isinstance(key, self._keytype): raise TypeError( f'Invalid key type {type(key)}; expected {self._keytype}') @@ -245,7 +248,7 @@ class BoundDictField(Generic[TK, T]): typedval: T = self.d_field.d_value.filter_output(self.d_data[key]) return typedval - def get(self, key: TK, default: Optional[T] = None) -> Optional[T]: + def get(self, key: TKey, default: Optional[T] = None) -> Optional[T]: """Get a value if present, or a default otherwise.""" if not isinstance(key, self._keytype): raise TypeError( @@ -256,18 +259,18 @@ class BoundDictField(Generic[TK, T]): typedval: T = self.d_field.d_value.filter_output(self.d_data[key]) return typedval - def __setitem__(self, key: TK, value: T) -> None: + def __setitem__(self, key: TKey, value: T) -> None: if not isinstance(key, self._keytype): raise TypeError("Expected str index.") self.d_data[key] = self.d_field.d_value.filter_input(value, error=True) - def __contains__(self, key: TK) -> bool: + def __contains__(self, key: TKey) -> bool: return key in self.d_data - def __delitem__(self, key: TK) -> None: + def __delitem__(self, key: TKey) -> None: del self.d_data[key] - def keys(self) -> List[TK]: + def keys(self) -> List[TKey]: """Return a list of our keys.""" return list(self.d_data.keys()) @@ -278,16 +281,16 @@ class BoundDictField(Generic[TK, T]): for value in self.d_data.values() ] - def items(self) -> List[Tuple[TK, T]]: + def items(self) -> List[Tuple[TKey, T]]: """Return a list of item/value pairs.""" return [(key, self.d_field.d_value.filter_output(value)) for key, value in self.d_data.items()] -class BoundCompoundListField(Generic[TC]): +class BoundCompoundListField(Generic[TCompound]): """A CompoundListField bound to its entity sub-data.""" - def __init__(self, field: CompoundListField[TC], d_data: List[Any]): + def __init__(self, field: CompoundListField[TCompound], d_data: List[Any]): self.d_field = field self.d_data = d_data self._i = 0 @@ -323,20 +326,20 @@ class BoundCompoundListField(Generic[TC]): if TYPE_CHECKING: @overload - def __getitem__(self, key: int) -> TC: + def __getitem__(self, key: int) -> TCompound: ... @overload - def __getitem__(self, key: slice) -> List[TC]: + def __getitem__(self, key: slice) -> List[TCompound]: ... def __getitem__(self, key: Any) -> Any: ... - def __next__(self) -> TC: + def __next__(self) -> TCompound: ... - def append(self) -> TC: + def append(self) -> TCompound: """Append and return a new field entry to the array.""" ... else: @@ -366,16 +369,16 @@ class BoundCompoundListField(Generic[TC]): self.d_field.d_value.get_default_data(), error=True)) return BoundCompoundValue(self.d_field.d_value, self.d_data[-1]) - def __iter__(self: TBL) -> TBL: + def __iter__(self: TBoundList) -> TBoundList: self._i = 0 return self -class BoundCompoundDictField(Generic[TK, TC]): +class BoundCompoundDictField(Generic[TKey, TCompound]): """A CompoundDictField bound to its entity sub-data.""" - def __init__(self, field: CompoundDictField[TK, TC], d_data: Dict[Any, - Any]): + def __init__(self, field: CompoundDictField[TKey, TCompound], + d_data: Dict[Any, Any]): self.d_field = field self.d_data = d_data @@ -408,16 +411,16 @@ class BoundCompoundDictField(Generic[TK, TC]): # would not be able to make sense of) if TYPE_CHECKING: - def __getitem__(self, key: TK) -> TC: + def __getitem__(self, key: TKey) -> TCompound: pass - def values(self) -> List[TC]: + def values(self) -> List[TCompound]: """Return a list of our values.""" - def items(self) -> List[Tuple[TK, TC]]: + def items(self) -> List[Tuple[TKey, TCompound]]: """Return key/value pairs for all dict entries.""" - def add(self, key: TK) -> TC: + def add(self, key: TKey) -> TCompound: """Add an entry into the dict, returning it. Any existing value is replaced.""" @@ -438,14 +441,14 @@ class BoundCompoundDictField(Generic[TK, TC]): return [(key, BoundCompoundValue(self.d_field.d_value, value)) for key, value in self.d_data.items()] - def add(self, key: TK) -> TC: + def add(self, key: TKey) -> TCompound: """Add an entry into the dict, returning it. Any existing value is replaced.""" if not isinstance(key, self.d_field.d_keytype): raise TypeError(f'expected key type {self.d_field.d_keytype};' f' got {type(key)}') - # push the entity default into data and then let it fill in + # Push the entity default into data and then let it fill in # any children/etc. self.d_data[key] = (self.d_field.d_value.filter_input( self.d_field.d_value.get_default_data(), error=True)) @@ -454,12 +457,12 @@ class BoundCompoundDictField(Generic[TK, TC]): def __len__(self) -> int: return len(self.d_data) - def __contains__(self, key: TK) -> bool: + def __contains__(self, key: TKey) -> bool: return key in self.d_data - def __delitem__(self, key: TK) -> None: + def __delitem__(self, key: TKey) -> None: del self.d_data[key] - def keys(self) -> List[TK]: + def keys(self) -> List[TKey]: """Return a list of our keys.""" return list(self.d_data.keys()) diff --git a/docs/ba_module.md b/docs/ba_module.md index d9a401c3..e1bc0a58 100644 --- a/docs/ba_module.md +++ b/docs/ba_module.md @@ -1,6 +1,6 @@ - -

last updated on 2019-12-14 for Ballistica version 1.5.0 build 20001

+ +

last updated on 2019-12-19 for Ballistica version 1.5.0 build 20001

This page documents the Python classes and functions in the 'ba' module, which are the ones most relevant to modding in Ballistica. If you come across something you feel should be included here or could be better explained, please let me know. Happy modding!

diff --git a/tests/test_bafoundation/test_entities.py b/tests/test_bafoundation/test_entities.py index dd0436a5..cceaf700 100644 --- a/tests/test_bafoundation/test_entities.py +++ b/tests/test_bafoundation/test_entities.py @@ -68,10 +68,10 @@ class EntityTest(entity.Entity): enumval = entity.Field('e', entity.EnumValue(EnumTest, default=None)) enumval2 = entity.Field( 'e2', entity.OptionalEnumValue(EnumTest, default=EnumTest.SECOND)) - compoundlist = entity.CompoundListField('l', CompoundTest()) slval = entity.ListField('sl', entity.StringValue()) tval2 = entity.Field('t2', entity.DateTimeValue()) str_int_dict = entity.DictField('sd', str, entity.IntValue()) + compoundlist = entity.CompoundListField('l', CompoundTest()) tdval = entity.CompoundDictField('td', str, CompoundTest()) fval2 = entity.Field('f2', entity.Float3Value()) @@ -98,7 +98,18 @@ def test_entity_values() -> None: ent.fval = True ent.fval = 1.0 - # Simple str/int dict field. + # Simple value list field. + assert not ent.slval + assert len(ent.slval) == 0 + with pytest.raises(TypeError): + ent.slval.append(1) # type: ignore + ent.slval.append('blah') + assert len(ent.slval) == 1 + assert list(ent.slval) == ['blah'] + with pytest.raises(TypeError): + ent.slval = ['foo', 'bar', 1] # type: ignore + + # Simple value dict field. assert 'foo' not in ent.str_int_dict with pytest.raises(TypeError): ent.str_int_dict[0] = 123 # type: ignore @@ -108,6 +119,25 @@ def test_entity_values() -> None: assert static_type_equals(ent.str_int_dict['foo'], int) assert ent.str_int_dict['foo'] == 123 + # Waaah; this works at runtime, but it seems that we'd need + # to have BoundDictField inherit from Mapping for mypy to accept this. + # (which seems to get a bit ugly, but may be worth revisiting) + # assert dict(ent.str_int_dict) == {'foo': 123} + + +# noinspection PyTypeHints +def test_entity_values_2() -> None: + """Test various entity assigns for value and type correctness.""" + + ent = EntityTest() + + # Compound value + assert static_type_equals(ent.grp, CompoundTest) + assert static_type_equals(ent.grp.isubval, int) + assert isinstance(ent.grp.isubval, int) + with pytest.raises(TypeError): + ent.grp.isubval = 'blah' # type: ignore + # Compound value inheritance. assert ent.grp2.isubval2 == 3453 assert ent.grp2.isubval == 34532 @@ -141,6 +171,67 @@ def test_entity_values() -> None: assert static_type_equals(ent.grp.compoundlist[0].subval, bool) +def test_field_copies() -> None: + """Test copying various values between fields.""" + ent1 = EntityTest() + ent2 = EntityTest() + + # Copying a simple value. + ent1.ival = 334 + ent2.ival = ent1.ival + assert ent2.ival == 334 + + # Copying a nested compound. + ent1.grp.isubval = 543 + ent2.grp = ent1.grp + assert ent2.grp.isubval == 543 + + # Type-checker currently allows this because both are Compounds + # but should fail at runtime since their subfield arrangement differs. + # reveal_type(ent1.grp.blah) + with pytest.raises(ValueError): + ent2.grp = ent1.grp2 + + # Copying a value list. + ent1.slval = ['foo', 'bar'] + assert ent1.slval == ['foo', 'bar'] + ent2.slval = ent1.slval + assert ent2.slval == ['foo', 'bar'] + + # Copying a value dict. + ent1.str_int_dict['tval'] = 987 + ent2.str_int_dict = ent1.str_int_dict + assert ent2.str_int_dict['tval'] == 987 + + # Copying a CompoundList + val = ent1.compoundlist.append() + val.isubval = 356 + assert ent1.compoundlist[0].isubval == 356 + assert len(ent1.compoundlist) == 1 + assert len(ent2.compoundlist) == 0 + ent2.compoundlist = ent1.compoundlist + assert ent2.compoundlist[0].isubval == 356 + assert len(ent2.compoundlist) == 1 + + +def test_field_access_from_type() -> None: + """Accessing fields through type objects should return the Field objs.""" + + ent = EntityTest() + + # Accessing fields through the type should return field objects + # instead of values. + assert static_type_equals(ent.ival, int) + assert isinstance(ent.ival, int) + mypytype = 'bafoundation.entity._field.Field[builtins.int*]' + assert static_type_equals(type(ent).ival, mypytype) + assert isinstance(type(ent).ival, entity.Field) + + # Accessing subtype on a nested compound field.. + assert static_type_equals(type(ent).compoundlist.d_value, CompoundTest) + assert isinstance(type(ent).compoundlist.d_value, CompoundTest) + + class EntityTestMixin(entity.EntityMixin, CompoundTest2): """A test entity created from a compound using a mixin class.""" diff --git a/tools/cloudtool b/tools/cloudtool index 413e3674..d73edffe 100755 --- a/tools/cloudtool +++ b/tools/cloudtool @@ -62,7 +62,7 @@ CLREND = '\033[0m' # End. CMD_LOGIN = 'login' CMD_LOGOUT = 'logout' -CMD_PUTASSETPACK = 'putassetpack' +CMD_ASSETPACK = 'assetpack' CMD_HELP = 'help' # Note to self: keep this synced with server-side logic. @@ -182,7 +182,7 @@ class AssetPackage: f'Invalid asset type for {assetpath} in {indexfilename}') assettype = AssetType(assettypestr) - print('Looking at asset:', assetpath, assetdata) + # print('Looking at asset:', assetpath, assetdata) package.assets[assetpath] = Asset(package, assettype, assetpath) return package @@ -198,10 +198,12 @@ class AssetPackage: def _get_asset_info(iasset: Asset) -> Tuple[Asset, Dict]: sha = hashlib.sha256() with open(iasset.filepath, 'rb') as infile: - sha.update(infile.read()) + filebytes = infile.read() + filesize = len(filebytes) + sha.update(filebytes) if not os.path.isfile(iasset.filepath): raise Exception(f'Asset file not found: "{iasset.filepath}"') - info_out: Dict = {'hash': sha.hexdigest()} + info_out: Dict = {'hash': sha.hexdigest(), 'size': filesize} return iasset, info_out # Use all procs to hash files for extra speedy goodness. @@ -246,8 +248,9 @@ class App: self.do_login() elif cmd == CMD_LOGOUT: self.do_logout() - elif cmd == CMD_PUTASSETPACK: - self.do_putassetpack() + elif (cmd == CMD_ASSETPACK and len(sys.argv) > 2 + and sys.argv[2] == 'put'): + self.do_assetpack_put() else: # For all other commands, simply pass them to the server verbatim. self.do_misc_command() @@ -332,31 +335,37 @@ class App: self._state.login_token = None print(f'{CLRGRN}Cloudtool is now logged out.{CLREND}') - def do_putassetpack(self) -> None: - """Run a putassetpack command.""" + def do_assetpack_put(self) -> None: + """Run an assetpackput command.""" - if len(sys.argv) != 3: + if len(sys.argv) != 4: raise CleanError('Expected a path to an assetpackage directory.') - path = Path(sys.argv[2]) + path = Path(sys.argv[3]) package = AssetPackage.load_from_disk(path) # Send the server a manifest of everything we've got locally. manifest = package.get_manifest() - print('SENDING PACKAGE MANIFEST:', manifest) - response = self._servercmd('putassetpackmanifest', {'m': manifest}) + response = self._servercmd('assetpackputmanifest', {'m': manifest}) - # The server should give us an upload id and a set of files it wants. + # The server should give us a version and a set of files it wants. # Upload each of those. upload_files: List[str] = response.data['upload_files'] assert isinstance(upload_files, list) assert all(isinstance(f, str) for f in upload_files) - self._putassetpack_upload(package, upload_files) + version = response.data['package_version'] + assert isinstance(version, str) + self._assetpack_put_upload(package, version, upload_files) - print('Asset upload successful!') + # Lastly, send a 'finish' command - this will prompt a response + # with info about the completed package. + _response = self._servercmd('assetpackputfinish', { + 'packageversion': version, + }) + # print(f'{CLRGRN}Created asset package: {version}{CLREND}') - def _putassetpack_upload(self, package: AssetPackage, - files: List[str]) -> None: + def _assetpack_put_upload(self, package: AssetPackage, version: str, + files: List[str]) -> None: # Upload the files one at a time. # (we can potentially do this in parallel in the future). @@ -374,9 +383,14 @@ class App: check=True) with open(gzpath, 'rb') as infile: putfiles: Dict = {'file': infile} - _response = self._servercmd('putassetpackupload', - {'path': asset.path}, - files=putfiles) + _response = self._servercmd( + 'assetpackputupload', + { + 'packageversion': version, + 'path': asset.path + }, + files=putfiles, + ) def do_misc_command(self) -> None: """Run a miscellaneous command.""" diff --git a/tools/efrotools/__init__.py b/tools/efrotools/__init__.py index 60fa73e9..b132f26a 100644 --- a/tools/efrotools/__init__.py +++ b/tools/efrotools/__init__.py @@ -141,11 +141,11 @@ def run(cmd: str) -> None: subprocess.run(cmd, shell=True, check=True) -def get_files_hash( - filenames: Sequence[Union[str, Path]], - extrahash: str = '', - int_only: bool = False, - hashtype: Union[Literal['md5'], Literal['sha256']] = 'md5') -> str: +# 1 +def get_files_hash(filenames: Sequence[Union[str, Path]], + extrahash: str = '', + int_only: bool = False, + hashtype: Literal['md5', 'sha256'] = 'md5') -> str: """Return a md5 hash for the given files.""" import hashlib if not isinstance(filenames, list): diff --git a/tools/efrotools/snippets.py b/tools/efrotools/snippets.py index f7faa9e8..c2e4cd2e 100644 --- a/tools/efrotools/snippets.py +++ b/tools/efrotools/snippets.py @@ -357,10 +357,20 @@ def sync_all() -> None: This assumes that there is a 'syncfull' and 'synclist' Makefile target under each project. """ + print(f'{CLRBLU}Updating formatting for all projects...{CLREND}') projects_str = os.environ.get('EFROTOOLS_SYNC_PROJECTS') if projects_str is None: - print('EFROTOOL_SYNC_PROJECTS is not defined.') - sys.exit(255) + raise CleanError('EFROTOOL_SYNC_PROJECTS is not defined.') + + # No matter what we're doing (even if just listing), run formatting + # in all projects before beginning. Otherwise if we do a sync and then + # a preflight we'll often wind up getting out-of-sync errors due to + # formatting changing after the sync. + for project in projects_str.split(':'): + cmd = f'cd "{project}" && make format' + print(cmd) + subprocess.run(cmd, shell=True, check=True) + if len(sys.argv) > 2 and sys.argv[2] == 'list': # List mode for project in projects_str.split(':'): diff --git a/tools/efrotools/statictest.py b/tools/efrotools/statictest.py index b23499bb..5bc598dc 100644 --- a/tools/efrotools/statictest.py +++ b/tools/efrotools/statictest.py @@ -28,7 +28,7 @@ import os import subprocess if TYPE_CHECKING: - from typing import Any, Type, Dict, Optional, List + from typing import Any, Type, Dict, Optional, List, Union # Global state: # We maintain a single temp dir where our mypy cache and our temp @@ -131,7 +131,13 @@ class StaticTestFile: # Parse this line as AST - we should find an assert # statement containing a static_type_equals() call # with 2 args. - tree = ast.parse(line[offset:]) + try: + tree = ast.parse(line[offset:]) + except Exception: + raise RuntimeError( + f"{self._filename} line {lineno+1}: unable to " + f"parse line (static_type_equals() call cannot" + f" be split across lines).") from None assert isinstance(tree, ast.Module) if (len(tree.body) != 1 or not isinstance(tree.body[0], ast.Assert)): @@ -165,13 +171,19 @@ class StaticTestFile: return '\n'.join(lines_out) + '\n' -def static_type_equals(value: Any, statictype: Type) -> bool: - """Check a type statically using mypy.""" +def static_type_equals(value: Any, statictype: Union[Type, str]) -> bool: + """Check a type statically using mypy. + + If a string is passed as statictype, it is checked against the mypy + output for an exact match. + If a type is passed, various filtering may apply when searching for + a match (for instance, if mypy outputs 'builtins.int*' it will match + the 'int' type passed in as statictype). + """ from inspect import getframeinfo, stack # We don't actually use there here; we pull them as strings from the src. del value - del statictype # Get the filename and line number of the calling function. caller = getframeinfo(stack()[1][0]) @@ -182,23 +194,36 @@ def static_type_equals(value: Any, statictype: Type) -> bool: _statictestfiles[filename] = StaticTestFile(filename) testfile = _statictestfiles[filename] - wanttype = testfile.linetypes_wanted[linenumber] mypytype = testfile.linetypes_mypy[linenumber] - # Do some filtering of Mypy types so we can compare to simple python ones. - # (ie: 'builtins.list[builtins.int*]' -> int) - # Note to self: perhaps we'd want a fallback form where we can pass a - # type as a string if we want to match the exact mypy value?... - mypytype = mypytype.replace('*', '') - mypytype = mypytype.replace('?', '') - mypytype = mypytype.replace('builtins.int', 'int') - mypytype = mypytype.replace('builtins.float', 'float') - mypytype = mypytype.replace('builtins.list', 'List') - mypytype = mypytype.replace('builtins.bool', 'bool') - mypytype = mypytype.replace('typing.Sequence', 'Sequence') + if isinstance(statictype, str): + # If they passed a string value as the statictype, + # do a comparison with the exact mypy value. + wanttype = statictype + else: + # If they passed a type object, things are a bit trickier because + # mypy's name for the type might not match the name we pass it in with. + # Try to do some filtering to minimize these differences... + wanttype = testfile.linetypes_wanted[linenumber] + del statictype - # temp3.FooClass -> FooClass - mypytype = mypytype.replace(testfile.modulename + '.', '') + # Do some filtering of Mypy types so we can compare + # to simple python ones. + + # (ie: 'builtins.list[builtins.int*]' -> int) + # Note to self: perhaps we'd want a fallback form where we can pass a + # type as a string if we want to match the exact mypy value?... + mypytype = mypytype.replace('*', '') + mypytype = mypytype.replace('?', '') + mypytype = mypytype.replace('builtins.int', 'int') + mypytype = mypytype.replace('builtins.float', 'float') + mypytype = mypytype.replace('builtins.list', 'List') + mypytype = mypytype.replace('builtins.bool', 'bool') + mypytype = mypytype.replace('typing.Sequence', 'Sequence') + + # So classes declared in the test file can be passed using base names. + # ie: temp3.FooClass -> FooClass + mypytype = mypytype.replace(testfile.modulename + '.', '') if wanttype != mypytype: print(f'Mypy type "{mypytype}" does not match '