diff --git a/.efrocachemap b/.efrocachemap index 6a6106a1..ad178d76 100644 --- a/.efrocachemap +++ b/.efrocachemap @@ -4056,26 +4056,26 @@ "build/assets/windows/Win32/ucrtbased.dll": "2def5335207d41b21b9823f6805997f1", "build/assets/windows/Win32/vc_redist.x86.exe": "b08a55e2e77623fe657bea24f223a3ae", "build/assets/windows/Win32/vcruntime140d.dll": "865b2af4d1e26a1a8073c89acb06e599", - "build/prefab/full/linux_arm64_gui/debug/ballisticakit": "10ca339552ccefcc1cf0458d77ddfc60", - "build/prefab/full/linux_arm64_gui/release/ballisticakit": "220a8bae9a55c6625972254dd7265029", - "build/prefab/full/linux_arm64_server/debug/dist/ballisticakit_headless": "54bde68d1f6c948a0fe1a3844b49fbc9", - "build/prefab/full/linux_arm64_server/release/dist/ballisticakit_headless": "d6908e5517980c48204c5d7d38b624b2", - "build/prefab/full/linux_x86_64_gui/debug/ballisticakit": "0552bfb41850409763c3eb0824629c4f", - "build/prefab/full/linux_x86_64_gui/release/ballisticakit": "3b92b9b7b6f25ec0b00ebdac6af8f7f7", - "build/prefab/full/linux_x86_64_server/debug/dist/ballisticakit_headless": "bf943bb02a696a97b8eb13173bd10381", - "build/prefab/full/linux_x86_64_server/release/dist/ballisticakit_headless": "733c06666e6ac2bd49a645814820ae70", - "build/prefab/full/mac_arm64_gui/debug/ballisticakit": "274f31c8955c9af9799edc72eef37184", - "build/prefab/full/mac_arm64_gui/release/ballisticakit": "ced3d5b6e818c2c5901e6373d40583ba", - "build/prefab/full/mac_arm64_server/debug/dist/ballisticakit_headless": "c745ec776c3fa19166fab17b91541c02", - "build/prefab/full/mac_arm64_server/release/dist/ballisticakit_headless": "cf3751677f46d248d251b566b737c3ca", - "build/prefab/full/mac_x86_64_gui/debug/ballisticakit": "ab74e096ad474d389c4e20cc78f22e43", - "build/prefab/full/mac_x86_64_gui/release/ballisticakit": "e54648fc2a899449baa081b55cb52588", - "build/prefab/full/mac_x86_64_server/debug/dist/ballisticakit_headless": "882935d0e1e2a9a36dfb7e0a3d4a5170", - "build/prefab/full/mac_x86_64_server/release/dist/ballisticakit_headless": "342b58d91ed50562f9f8d8163f5f4441", - "build/prefab/full/windows_x86_gui/debug/BallisticaKit.exe": "591cf3434d9ac490d416c9dcc60f051f", - "build/prefab/full/windows_x86_gui/release/BallisticaKit.exe": "4ef7a5bbfef88bf2174a0001aaf9532d", - "build/prefab/full/windows_x86_server/debug/dist/BallisticaKitHeadless.exe": "16253a3ef3ccc96250eb5a311caf14fe", - "build/prefab/full/windows_x86_server/release/dist/BallisticaKitHeadless.exe": "90ab514825fbb1e5c2a88297b19d95d2", + "build/prefab/full/linux_arm64_gui/debug/ballisticakit": "279d063ae54801d02d5f9739a8cf2fcb", + "build/prefab/full/linux_arm64_gui/release/ballisticakit": "21fb39e8d8aba6d6027f3465ab5fbc6b", + "build/prefab/full/linux_arm64_server/debug/dist/ballisticakit_headless": "c1dc7f237469dd36deffe5414a81a5c2", + "build/prefab/full/linux_arm64_server/release/dist/ballisticakit_headless": "13436c4ecb012fa616ab3b52d765a4a8", + "build/prefab/full/linux_x86_64_gui/debug/ballisticakit": "481e76f0d618ea949640779db8089e46", + "build/prefab/full/linux_x86_64_gui/release/ballisticakit": "6452a4a354d36a5618275719046c45c0", + "build/prefab/full/linux_x86_64_server/debug/dist/ballisticakit_headless": "99ea8e8d810194730a58c1f61baf6bd4", + "build/prefab/full/linux_x86_64_server/release/dist/ballisticakit_headless": "9206a652925c58d5f54ac3b19202568b", + "build/prefab/full/mac_arm64_gui/debug/ballisticakit": "de160cabd194d8e1a08945e2a990c6e7", + "build/prefab/full/mac_arm64_gui/release/ballisticakit": "45f1d057b67eea2b8937c0987f4600df", + "build/prefab/full/mac_arm64_server/debug/dist/ballisticakit_headless": "05fd1a0c61f766d01fdb4d8a6f21884a", + "build/prefab/full/mac_arm64_server/release/dist/ballisticakit_headless": "652bb06c069ea03d82b20d94a1c0d8b0", + "build/prefab/full/mac_x86_64_gui/debug/ballisticakit": "90e139a76507c00deff3ea1dcf569935", + "build/prefab/full/mac_x86_64_gui/release/ballisticakit": "7314fa7928a4714dfb2c44bd0e976878", + "build/prefab/full/mac_x86_64_server/debug/dist/ballisticakit_headless": "eb68e8ab4ee7c9bb9748ba72d92424ce", + "build/prefab/full/mac_x86_64_server/release/dist/ballisticakit_headless": "69c56cc9b9a373953008c5a3742c514a", + "build/prefab/full/windows_x86_gui/debug/BallisticaKit.exe": "ff043c91883c2c3fb71f64b744774232", + "build/prefab/full/windows_x86_gui/release/BallisticaKit.exe": "014cb8642e6fae3bf542117433bea233", + "build/prefab/full/windows_x86_server/debug/dist/BallisticaKitHeadless.exe": "f640bd3d830548bf42960643f89a7564", + "build/prefab/full/windows_x86_server/release/dist/BallisticaKitHeadless.exe": "1a340e7924157c2e84b0cd3c3ac84a6b", "build/prefab/lib/linux_arm64_gui/debug/libballisticaplus.a": "11668f7191dc8d4e2b03db15ff2a4241", "build/prefab/lib/linux_arm64_gui/release/libballisticaplus.a": "b190199ce8b8fe122993d14fb7e8e813", "build/prefab/lib/linux_arm64_server/debug/libballisticaplus.a": "11668f7191dc8d4e2b03db15ff2a4241", @@ -4092,14 +4092,14 @@ "build/prefab/lib/mac_x86_64_gui/release/libballisticaplus.a": "4fab22abc481a97e884cce9d67b35c6b", "build/prefab/lib/mac_x86_64_server/debug/libballisticaplus.a": "7c536b056d78148a302c975bf9238118", "build/prefab/lib/mac_x86_64_server/release/libballisticaplus.a": "4fab22abc481a97e884cce9d67b35c6b", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.lib": "be00cf9781b1c86439452c4ba0999c8d", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.pdb": "5803cfddbaf5e09f513aded1a8fc71f5", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.lib": "b682b559cef053626ee8ee9da79ad266", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.pdb": "8d467c3b01c4b2fd4977afd0aabb2bd0", - "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.lib": "748545de892e5ed6556ba0b05f9a0e9d", - "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.pdb": "2671eae8ad8f42eb272ffed0fc8e93df", - "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.lib": "23712747790fbd36e33b9e59364accb7", - "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.pdb": "1832b257d7cd01ca68b7b99cbb62bfa6", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.lib": "cf2978894cde54eed829e1c5b641c656", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.pdb": "a9a883ae6660b516ab3da8202ecf8afd", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.lib": "dc18e0fbb1c7c42a1d2ae7dbe8104fc9", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.pdb": "2b20efd81ca876dee30ec64917ae53fb", + "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.lib": "093edb4112d70c03032d938d345ff0be", + "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.pdb": "286e901c3f7395b790213295586ba619", + "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.lib": "2aae411134c4eb841f28dc932baba0d3", + "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.pdb": "e9989960860d3371f4b17eec3d6c098a", "src/assets/ba_data/python/babase/_mgen/__init__.py": "f885fed7f2ed98ff2ba271f9dbe3391c", "src/assets/ba_data/python/babase/_mgen/enums.py": "b611c090513a21e2fe90e56582724e9d", "src/ballistica/base/mgen/pyembed/binding_base.inc": "72bfed2cce8ff19741989dec28302f3f", diff --git a/CHANGELOG.md b/CHANGELOG.md index a0910474..fa2671b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,9 @@ -### 1.7.34 (build 21801, api 8, 2024-04-21) +### 1.7.34 (build 21802, api 8, 2024-04-23) - Bumped Python version from 3.11 to 3.12 for all builds and project tools. One of the things this means is that we can use `typing.override` instead of the - `typing_extensions` version so the annoying workaround of installing + `typing_extensions` version so the annoying requirement of installing `typing_extensions` first thing when setting up the repo introduced a few - versions back is finally no longer needed. + versions back is finally no longer a thing. - The project now maintains its own Python virtual environment in `.venv` where it automatically installs whatever Python packages it needs instead of asking the user to do so in their own environment. This should greatly simplify @@ -28,6 +28,7 @@ does not and you get errors. - `_bascenev1.protocol_version()` now properly throws an exception if called while scene-v1 is not active. +- The `efro.dataclassio` system now supports `datetime.timedelta` values. ### 1.7.33 (build 21795, api 8, 2024-03-24) - Stress test input-devices are now a bit smarter; they won't press any buttons diff --git a/Makefile b/Makefile index c5129fed..391f058e 100644 --- a/Makefile +++ b/Makefile @@ -149,13 +149,13 @@ meta-clean: # Remove ALL files and directories that aren't managed by git (except for a # few things such as localconfig.json). -clean: +clean: prereqs $(CHECK_CLEAN_SAFETY) rm -rf build # Kill this ourself; can confuse git if contains other repos. git clean -dfx $(ROOT_CLEAN_IGNORES) # Show what clean would delete without actually deleting it. -clean-list: +clean-list: prereqs $(CHECK_CLEAN_SAFETY) @echo Would remove build # Handle this part ourself; can confuse git. git clean -dnx $(ROOT_CLEAN_IGNORES) @@ -752,12 +752,12 @@ update-check: prereqs-pre-update ################################################################################ # Run formatting on all files in the project considered 'dirty'. -format: +format: prereqs @$(MAKE) -j$(CPUS) format-code format-scripts format-makefile @$(PCOMMANDBATCH) echo BLD Formatting complete for $(notdir $(CURDIR))! # Same but always formats; ignores dirty state. -format-full: +format-full: prereqs @$(MAKE) -j$(CPUS) format-code-full format-scripts-full format-makefile @$(PCOMMANDBATCH) echo BLD Formatting complete for $(notdir $(CURDIR))! @@ -933,6 +933,7 @@ test-rpc: preflight: @$(MAKE) format @$(MAKE) update + @$(MAKE) -j$(CPUS) py_check_prereqs # Needs to be done explicitly first. @$(MAKE) -j$(CPUS) cpplint pylint mypy test @$(PCOMMANDBATCH) echo SGRN BLD PREFLIGHT SUCCESSFUL! @@ -940,6 +941,7 @@ preflight: preflight-full: @$(MAKE) format-full @$(MAKE) update + @$(MAKE) -j$(CPUS) py_check_prereqs # Needs to be done explicitly first. @$(MAKE) -j$(CPUS) cpplint-full pylint-full mypy-full test-full @$(PCOMMANDBATCH) echo SGRN BLD PREFLIGHT SUCCESSFUL! @@ -947,6 +949,7 @@ preflight-full: preflight2: @$(MAKE) format @$(MAKE) update + @$(MAKE) -j$(CPUS) py_check_prereqs # Needs to be done explicitly first. @$(MAKE) -j$(CPUS) cpplint pylint mypy test @$(PCOMMANDBATCH) echo SGRN BLD PREFLIGHT SUCCESSFUL! @@ -954,6 +957,7 @@ preflight2: preflight2-full: @$(MAKE) format-full @$(MAKE) update + @$(MAKE) -j$(CPUS) py_check_prereqs # Needs to be done explicitly first. @$(MAKE) -j$(CPUS) cpplint-full pylint-full mypy-full test-full @$(PCOMMANDBATCH) echo SGRN BLD PREFLIGHT SUCCESSFUL! diff --git a/src/assets/ba_data/python/baenv.py b/src/assets/ba_data/python/baenv.py index 455808d3..886e0cff 100644 --- a/src/assets/ba_data/python/baenv.py +++ b/src/assets/ba_data/python/baenv.py @@ -52,7 +52,7 @@ if TYPE_CHECKING: # Build number and version of the ballistica binary we expect to be # using. -TARGET_BALLISTICA_BUILD = 21801 +TARGET_BALLISTICA_BUILD = 21802 TARGET_BALLISTICA_VERSION = '1.7.34' diff --git a/src/ballistica/shared/ballistica.cc b/src/ballistica/shared/ballistica.cc index 13076162..770845b4 100644 --- a/src/ballistica/shared/ballistica.cc +++ b/src/ballistica/shared/ballistica.cc @@ -39,7 +39,7 @@ auto main(int argc, char** argv) -> int { namespace ballistica { // These are set automatically via script; don't modify them here. -const int kEngineBuildNumber = 21801; +const int kEngineBuildNumber = 21802; const char* kEngineVersion = "1.7.34"; const int kEngineApiVersion = 8; diff --git a/tests/test_efro/test_dataclassio.py b/tests/test_efro/test_dataclassio.py index 64125594..38642ce1 100644 --- a/tests/test_efro/test_dataclassio.py +++ b/tests/test_efro/test_dataclassio.py @@ -101,6 +101,7 @@ def test_assign() -> None: dictval: dict[int, str] = field(default_factory=dict) tupleval: tuple[int, str, bool] = (1, 'foo', False) datetimeval: datetime.datetime | None = None + timedeltaval: datetime.timedelta | None = None class _TestClass2: pass @@ -116,10 +117,10 @@ def test_assign() -> None: dataclass_from_dict(_TestClass, None) # type: ignore now = utc_now() + tdelta = datetime.timedelta(days=123, seconds=456, microseconds=789) - # A dict containing *ALL* values should match what we - # get when creating a dataclass and then converting back - # to a dict. + # A dict containing *ALL* values should exactly match what we get + # when creating a dataclass and then converting back to a dict. dict1 = { 'ival': 1, 'sval': 'foo', @@ -156,6 +157,7 @@ def test_assign() -> None: now.second, now.microsecond, ], + 'timedeltaval': [tdelta.days, tdelta.seconds, tdelta.microseconds], } dc1 = dataclass_from_dict(_TestClass, dict1) assert dataclass_to_dict(dc1) == dict1 diff --git a/tools/batools/appmodule.py b/tools/batools/appmodule.py index 5e6dc721..f25b80aa 100755 --- a/tools/batools/appmodule.py +++ b/tools/batools/appmodule.py @@ -24,7 +24,8 @@ def generate_app_module( # pylint: disable=too-many-statements import textwrap - from efrotools import replace_section, getprojectconfig + from efrotools.util import replace_section + from efrotools.project import getprojectconfig out = '' diff --git a/tools/batools/assetsmakefile.py b/tools/batools/assetsmakefile.py index c477360b..75f993a9 100755 --- a/tools/batools/assetsmakefile.py +++ b/tools/batools/assetsmakefile.py @@ -415,7 +415,7 @@ def generate_assets_makefile( ) -> dict[str, str]: """Main script entry point.""" # pylint: disable=too-many-locals - from efrotools import getprojectconfig + from efrotools.project import getprojectconfig from pathlib import Path public = getprojectconfig(Path(projroot))['public'] diff --git a/tools/batools/metamakefile.py b/tools/batools/metamakefile.py index 1d79a098..cd3d2b4c 100755 --- a/tools/batools/metamakefile.py +++ b/tools/batools/metamakefile.py @@ -14,7 +14,7 @@ from typing import TYPE_CHECKING from dataclasses import dataclass from efro.error import CleanError -from efrotools import getprojectconfig +from efrotools.project import getprojectconfig if TYPE_CHECKING: pass diff --git a/tools/batools/pcommands.py b/tools/batools/pcommands.py index b8eca720..757876f7 100644 --- a/tools/batools/pcommands.py +++ b/tools/batools/pcommands.py @@ -126,7 +126,7 @@ def lazy_increment_build() -> None: import subprocess from efro.terminal import Clr from efro.error import CleanError - from efrotools import get_files_hash + from efrotools.util import get_files_hash from efrotools.code import get_code_filenames pcommand.disallow_in_batch() @@ -221,7 +221,7 @@ def androidaddr() -> None: def push_ipa() -> None: """Construct and push ios IPA for testing.""" - from efrotools import extract_arg + from efrotools.util import extract_arg import efrotools.ios pcommand.disallow_in_batch() @@ -493,7 +493,7 @@ def warm_start_asset_build() -> None: import subprocess from pathlib import Path - from efrotools import getprojectconfig + from efrotools.project import getprojectconfig from efro.error import CleanError pcommand.disallow_in_batch() diff --git a/tools/batools/pcommands2.py b/tools/batools/pcommands2.py index bfa18a9e..853c9752 100644 --- a/tools/batools/pcommands2.py +++ b/tools/batools/pcommands2.py @@ -124,7 +124,7 @@ def py_examine() -> None: """Run a python examination at a given point in a given file.""" import os from pathlib import Path - import efrotools + import efrotools.emacs pcommand.disallow_in_batch() @@ -153,7 +153,7 @@ def py_examine() -> None: sys.path.append(scriptsdir) if toolsdir not in sys.path: sys.path.append(toolsdir) - efrotools.py_examine( + efrotools.emacs.py_examine( pcommand.PROJROOT, filename, line, column, selection, operation ) diff --git a/tools/batools/project/_checks.py b/tools/batools/project/_checks.py index f1612d79..ecf3abaa 100755 --- a/tools/batools/project/_checks.py +++ b/tools/batools/project/_checks.py @@ -587,7 +587,7 @@ def check_sync_states(self: ProjectUpdater) -> None: def check_misc(self: ProjectUpdater) -> None: """Check misc project stuff.""" - from efrotools import readfile, replace_exact + from efrotools.util import readfile, replace_exact # Make sure we're set to prod master server. (but ONLY when # checking; still want to be able to run updates). diff --git a/tools/batools/project/_updater.py b/tools/batools/project/_updater.py index 885c0738..31959b55 100755 --- a/tools/batools/project/_updater.py +++ b/tools/batools/project/_updater.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import TYPE_CHECKING from dataclasses import dataclass -from efrotools import getprojectconfig, getlocalconfig +from efrotools.project import getprojectconfig, getlocalconfig from efro.error import CleanError from efro.terminal import Clr diff --git a/tools/batools/resourcesmakefile.py b/tools/batools/resourcesmakefile.py index b865a1fb..4768df5a 100755 --- a/tools/batools/resourcesmakefile.py +++ b/tools/batools/resourcesmakefile.py @@ -57,7 +57,7 @@ class ResourcesMakefileGenerator: existing_data: str, projname: str, ) -> None: - from efrotools import getprojectconfig + from efrotools.project import getprojectconfig self.public = getprojectconfig(Path(projroot))['public'] assert isinstance(self.public, bool) diff --git a/tools/batools/spinoff/_context.py b/tools/batools/spinoff/_context.py index 849a1ce6..d2516613 100644 --- a/tools/batools/spinoff/_context.py +++ b/tools/batools/spinoff/_context.py @@ -15,7 +15,8 @@ from pathlib import Path from typing import TYPE_CHECKING, assert_never from efrotools.code import format_python_str, format_cpp_str -from efrotools import getprojectconfig, replace_exact +from efrotools.project import getprojectconfig +from efrotools.util import replace_exact from efro.error import CleanError from efro.terminal import Clr from efro.util import timedelta_str @@ -693,7 +694,7 @@ class SpinoffContext: def _generate_env_hash(self) -> None: # pylint: disable=cyclic-import - from efrotools import get_files_hash + from efrotools.util import get_files_hash # noinspection PyUnresolvedReferences import batools.spinoff diff --git a/tools/batools/spinoff/_main.py b/tools/batools/spinoff/_main.py index eeb08eef..90284ec7 100644 --- a/tools/batools/spinoff/_main.py +++ b/tools/batools/spinoff/_main.py @@ -13,7 +13,7 @@ from typing import assert_never, TYPE_CHECKING from efro.error import CleanError from efro.terminal import Clr -from efrotools import replace_exact +from efrotools.util import replace_exact from batools.spinoff._context import SpinoffContext @@ -65,19 +65,8 @@ def _main() -> None: _print_available_commands() return - dst_root = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) - - # Determine our src project based on our tools/spinoff symlink. - # If its not a link it means we ARE a src project. - dst_spinoff_path = os.path.join(dst_root, 'tools', 'spinoff') - if os.path.islink(dst_spinoff_path): - src_root = os.path.abspath( - os.path.join( - os.path.dirname(os.path.realpath(dst_spinoff_path)), '..' - ) - ) - else: - src_root = None + src_root = os.environ['BA_SPINOFF_SRC_ROOT'] + dst_root = os.environ.get('BA_SPINOFF_DST_ROOT') single_run_mode: SpinoffContext.Mode | None = None @@ -102,14 +91,14 @@ def _main() -> None: elif cmd is Command.BACKPORT: _do_backport(src_root, dst_root) elif cmd is Command.FEATURE_SET_LIST: - _do_featuresets(dst_root) + _do_featuresets(src_root) elif cmd is Command.CREATE: _do_create(src_root, dst_root) elif cmd is Command.ADD_SUBMODULE_PARENT: - from efrotools import getprojectconfig + from efrotools.project import getprojectconfig - public = getprojectconfig(Path(dst_root))['public'] - _do_add_submodule_parent(dst_root, is_new=False, public=public) + public = getprojectconfig(Path(src_root))['public'] + _do_add_submodule_parent(src_root, is_new=False, public=public) elif cmd is Command.FEATURE_SET_COPY: _do_featureset_copy() elif cmd is Command.FEATURE_SET_DELETE: @@ -118,13 +107,13 @@ def _main() -> None: assert_never(cmd) if single_run_mode is not None: - from efrotools import extract_flag + from efrotools.util import extract_flag args = sys.argv[2:] force = extract_flag(args, '--force') verbose = extract_flag(args, '--verbose') print_full_lists = extract_flag(args, '--full') - if src_root is None: + if dst_root is None: if '--soft' in sys.argv: return raise CleanError( @@ -155,16 +144,14 @@ def _main() -> None: ).run() -def _do_create(src_root: str | None, dst_root: str) -> None: +def _do_create(src_root: str, dst_root: str | None) -> None: # pylint: disable=too-many-locals, cyclic-import - from efrotools import extract_arg, extract_flag + from efrotools.util import extract_arg, extract_flag from efrotools.code import format_python_str - from efrotools import getprojectconfig + from efrotools.project import getprojectconfig import batools.spinoff - # Note: in our case dst_root is actually what becomes the src project - # should clean up these var names to make that clearer. - if src_root is not None: + if dst_root is not None: raise CleanError('This only works on src projects.') args = sys.argv[2:] @@ -224,7 +211,7 @@ def _do_create(src_root: str | None, dst_root: str) -> None: template, '# __SRC_FEATURE_SETS__', format_python_str( - projroot=dst_root, code=f'ctx.src_feature_sets = {featuresets!r}' + projroot=src_root, code=f'ctx.src_feature_sets = {featuresets!r}' ), ) @@ -237,7 +224,7 @@ def _do_create(src_root: str | None, dst_root: str) -> None: # on git so its best to always do this. subprocess.run(['git', 'init'], cwd=path, check=True, capture_output=True) - public = getprojectconfig(Path(dst_root))['public'] + public = getprojectconfig(Path(src_root))['public'] if submodule_parent: _do_add_submodule_parent(path, is_new=True, public=public) @@ -246,7 +233,7 @@ def _do_create(src_root: str | None, dst_root: str) -> None: [ 'ln', '-s', - os.path.join(dst_root, 'tools', 'spinoff'), + os.path.join(src_root, 'tools', 'spinoff'), os.path.join(path, 'tools'), ], check=True, @@ -330,7 +317,7 @@ def _do_featureset_delete() -> None: def _do_featureset_copy() -> None: # pylint: disable=too-many-locals - from efrotools import extract_flag + from efrotools.util import extract_flag from batools.featureset import FeatureSet @@ -514,9 +501,10 @@ def _do_featureset_copy_dir( ) -def _do_override(src_root: str | None, dst_root: str) -> None: - if src_root is None: +def _do_override(src_root: str, dst_root: str | None) -> None: + if dst_root is None: raise CleanError('This only works on dst projects.') + override_paths = [os.path.abspath(p) for p in sys.argv[2:]] if not override_paths: raise RuntimeError('Expected at least one path arg.') @@ -549,8 +537,8 @@ def _do_override(src_root: str | None, dst_root: str) -> None: SpinoffContext(src_root, dst_root, SpinoffContext.Mode.UPDATE).run() -def _do_backport(src_root: str | None, dst_root: str) -> None: - if src_root is None: +def _do_backport(src_root: str, dst_root: str | None) -> None: + if dst_root is None: raise CleanError('This only works on dst projects.') args = sys.argv[2:] auto = '--auto' in args diff --git a/tools/batools/spinoff/_test.py b/tools/batools/spinoff/_test.py index 3587b7cb..313d22b5 100644 --- a/tools/batools/spinoff/_test.py +++ b/tools/batools/spinoff/_test.py @@ -20,7 +20,8 @@ def spinoff_test(args: list[str]) -> None: import subprocess from batools.featureset import FeatureSet - from efrotools import extract_flag, getprojectconfig + from efrotools.util import extract_flag + from efrotools.project import getprojectconfig from efro.terminal import Clr from efro.error import CleanError diff --git a/tools/batools/staging.py b/tools/batools/staging.py index a63436d0..11d2a158 100755 --- a/tools/batools/staging.py +++ b/tools/batools/staging.py @@ -12,7 +12,7 @@ from functools import partial from typing import TYPE_CHECKING from efro.terminal import Clr -from efrotools import extract_arg, extract_flag +from efrotools.util import extract_arg, extract_flag from efrotools.pyver import PYVER if TYPE_CHECKING: @@ -750,7 +750,7 @@ def _stage_server_file( ) -> None: """Stage files for the server environment with some filtering.""" import batools.build - from efrotools import replace_exact + from efrotools.util import replace_exact if mode not in ('debug', 'release'): raise RuntimeError( diff --git a/tools/batools/toplevelmakefile.py b/tools/batools/toplevelmakefile.py index 77579364..2e98ce92 100755 --- a/tools/batools/toplevelmakefile.py +++ b/tools/batools/toplevelmakefile.py @@ -14,7 +14,7 @@ if TYPE_CHECKING: def generate_top_level_makefile(projroot: str, existing_data: str) -> str: """Main script entry point.""" - from efrotools import getprojectconfig + from efrotools.project import getprojectconfig from pathlib import Path public = getprojectconfig(Path(projroot))['public'] diff --git a/tools/efro/dataclassio/_inputter.py b/tools/efro/dataclassio/_inputter.py index 81992a78..41588e98 100644 --- a/tools/efro/dataclassio/_inputter.py +++ b/tools/efro/dataclassio/_inputter.py @@ -193,6 +193,9 @@ class _Inputter: if issubclass(origin, datetime.datetime): return self._datetime_from_input(cls, fieldpath, value, ioattrs) + if issubclass(origin, datetime.timedelta): + return self._timedelta_from_input(cls, fieldpath, value, ioattrs) + if origin is bytes: return self._bytes_from_input(origin, fieldpath, value) @@ -634,3 +637,23 @@ class _Inputter: if ioattrs is not None: ioattrs.validate_datetime(out, fieldpath) return out + + def _timedelta_from_input( + self, cls: type, fieldpath: str, value: Any, ioattrs: IOAttrs | None + ) -> Any: + del ioattrs # Unused. + # We expect a list of 3 ints. + if type(value) is not list: + raise TypeError( + f'Invalid input value for "{fieldpath}" on "{cls.__name__}";' + f' expected a list, got a {type(value).__name__}' + ) + if len(value) != 3 or not all(isinstance(x, int) for x in value): + raise ValueError( + f'Invalid input value for "{fieldpath}" on "{cls.__name__}";' + f' expected a list of 3 ints, got {[type(v) for v in value]}.' + ) + out = datetime.timedelta( + days=value[0], seconds=value[1], microseconds=value[2] + ) + return out diff --git a/tools/efro/dataclassio/_outputter.py b/tools/efro/dataclassio/_outputter.py index 050f1cde..56417ddc 100644 --- a/tools/efro/dataclassio/_outputter.py +++ b/tools/efro/dataclassio/_outputter.py @@ -454,6 +454,17 @@ class _Outputter: if self._create else None ) + if issubclass(origin, datetime.timedelta): + if not isinstance(value, origin): + raise TypeError( + f'Expected a {origin} for {fieldpath};' + f' found a {type(value)}.' + ) + return ( + [value.days, value.seconds, value.microseconds] + if self._create + else None + ) if origin is bytes: return self._process_bytes(cls, fieldpath, value) diff --git a/tools/efro/dataclassio/_prep.py b/tools/efro/dataclassio/_prep.py index 53055dda..8321dbdc 100644 --- a/tools/efro/dataclassio/_prep.py +++ b/tools/efro/dataclassio/_prep.py @@ -420,6 +420,10 @@ class PrepSession: if issubclass(origin, datetime.datetime): return + # We support datetime.timedelta. + if issubclass(origin, datetime.timedelta): + return + if dataclasses.is_dataclass(origin): self.prep_dataclass(origin, recursion_level=recursion_level + 1) return diff --git a/tools/efrotools/__init__.py b/tools/efrotools/__init__.py index 5b678587..8f608a37 100644 --- a/tools/efrotools/__init__.py +++ b/tools/efrotools/__init__.py @@ -5,336 +5,3 @@ This stuff can be a bit more sloppy/loosey-goosey since it is not used in live client or server code. """ - -# FIXME: should migrate everything here into submodules since this adds -# overhead to anything importing from any efrotools submodule. - -from __future__ import annotations - -import os -import json -from pathlib import Path -from typing import TYPE_CHECKING, overload - - -if TYPE_CHECKING: - from typing import Sequence, Any, Literal - -# Cache these since we may repeatedly fetch these in batch mode. -_g_project_configs: dict[str, dict[str, Any]] = {} -_g_local_configs: dict[str, dict[str, Any]] = {} - - -def explicit_bool(value: bool) -> bool: - """Simply return input value; can avoid unreachable-code type warnings.""" - return value - - -def getlocalconfig(projroot: Path | str) -> dict[str, Any]: - """Return a project's localconfig contents (or default if missing).""" - projrootstr = str(projroot) - if projrootstr not in _g_local_configs: - localconfig: dict[str, Any] - - # Allow overriding path via env var. - path = os.environ.get('EFRO_LOCALCONFIG_PATH') - if path is None: - path = 'config/localconfig.json' - - try: - with open(Path(projroot, path), encoding='utf-8') as infile: - localconfig = json.loads(infile.read()) - except FileNotFoundError: - localconfig = {} - _g_local_configs[projrootstr] = localconfig - - return _g_local_configs[projrootstr] - - -def getprojectconfig(projroot: Path | str) -> dict[str, Any]: - """Return a project's projectconfig contents (or default if missing).""" - projrootstr = str(projroot) - if projrootstr not in _g_project_configs: - config: dict[str, Any] - try: - with open( - Path(projroot, 'config/projectconfig.json'), encoding='utf-8' - ) as infile: - config = json.loads(infile.read()) - except FileNotFoundError: - config = {} - _g_project_configs[projrootstr] = config - return _g_project_configs[projrootstr] - - -def setprojectconfig(projroot: Path | str, config: dict[str, Any]) -> None: - """Set the project config contents.""" - projrootstr = str(projroot) - _g_project_configs[projrootstr] = config - os.makedirs(Path(projroot, 'config'), exist_ok=True) - with Path(projroot, 'config/projectconfig.json').open( - 'w', encoding='utf-8' - ) as outfile: - outfile.write(json.dumps(config, indent=2)) - - -def extract_flag(args: list[str], name: str) -> bool: - """Given a list of args and a flag name, returns whether it is present. - - The arg flag, if present, is removed from the arg list. - """ - from efro.error import CleanError - - count = args.count(name) - if count > 1: - raise CleanError(f'Flag {name} passed multiple times.') - if not count: - return False - args.remove(name) - return True - - -@overload -def extract_arg( - args: list[str], name: str, required: Literal[False] = False -) -> str | None: ... - - -@overload -def extract_arg(args: list[str], name: str, required: Literal[True]) -> str: ... - - -def extract_arg( - args: list[str], name: str, required: bool = False -) -> str | None: - """Given a list of args and an arg name, returns a value. - - The arg flag and value are removed from the arg list. - raises CleanErrors on any problems. - """ - from efro.error import CleanError - - count = args.count(name) - if not count: - if required: - raise CleanError(f'Required argument {name} not passed.') - return None - - if count > 1: - raise CleanError(f'Arg {name} passed multiple times.') - - argindex = args.index(name) - if argindex + 1 >= len(args): - raise CleanError(f'No value passed after {name} arg.') - - val = args[argindex + 1] - del args[argindex : argindex + 2] - - return val - - -def replace_section( - text: str, - begin_marker: str, - end_marker: str, - replace_text: str = '', - keep_markers: bool = False, - error_if_missing: bool = True, -) -> str: - """Replace all text between two marker strings (including the markers).""" - if begin_marker not in text: - if error_if_missing: - raise RuntimeError(f"Marker not found in text: '{begin_marker}'.") - return text - splits = text.split(begin_marker) - if len(splits) != 2: - raise RuntimeError( - f"Expected one marker '{begin_marker}'" - f'; found {text.count(begin_marker)}.' - ) - before_begin, after_begin = splits - splits = after_begin.split(end_marker) - if len(splits) != 2: - raise RuntimeError( - f"Expected one marker '{end_marker}'" - f'; found {text.count(end_marker)}.' - ) - _before_end, after_end = splits - if keep_markers: - replace_text = f'{begin_marker}{replace_text}{end_marker}' - return f'{before_begin}{replace_text}{after_end}' - - -def readfile(path: str | Path) -> str: - """Read a utf-8 text file into a string.""" - with open(path, encoding='utf-8') as infile: - return infile.read() - - -def writefile(path: str | Path, txt: str) -> None: - """Write a string to a utf-8 text file.""" - with open(path, 'w', encoding='utf-8') as outfile: - outfile.write(txt) - - -def replace_exact( - opstr: str, old: str, new: str, count: int = 1, label: str | None = None -) -> str: - """Replace text ensuring that exactly x occurrences are replaced. - - Useful when filtering data in some predefined way to ensure the original - has not changed. - """ - found = opstr.count(old) - label_str = f' in {label}' if label is not None else '' - if found != count: - raise RuntimeError( - f'Expected {count} string occurrence(s){label_str};' - f' found {found}. String: {repr(old)}' - ) - return opstr.replace(old, new) - - -def get_files_hash( - filenames: Sequence[str | Path], - extrahash: str = '', - int_only: bool = False, - hashtype: Literal['md5', 'sha256'] = 'md5', -) -> str: - """Return a hash for the given files.""" - import hashlib - - if not isinstance(filenames, list): - raise RuntimeError(f'Expected a list; got a {type(filenames)}.') - if TYPE_CHECKING: - # Help Mypy infer the right type for this. - hashobj = hashlib.md5() - else: - hashobj = getattr(hashlib, hashtype)() - for fname in filenames: - with open(fname, 'rb') as infile: - while True: - data = infile.read(2**20) - if not data: - break - hashobj.update(data) - hashobj.update(extrahash.encode()) - - if int_only: - return str(int.from_bytes(hashobj.digest(), byteorder='big')) - - return hashobj.hexdigest() - - -def get_string_hash( - value: str, - int_only: bool = False, - hashtype: Literal['md5', 'sha256'] = 'md5', -) -> str: - """Return a hash for the given files.""" - import hashlib - - if not isinstance(value, str): - raise TypeError('Expected a str.') - if TYPE_CHECKING: - # Help Mypy infer the right type for this. - hashobj = hashlib.md5() - else: - hashobj = getattr(hashlib, hashtype)() - hashobj.update(value.encode()) - - if int_only: - return str(int.from_bytes(hashobj.digest(), byteorder='big')) - - return hashobj.hexdigest() - - -def _py_symbol_at_column(line: str, col: int) -> str: - start = col - while start > 0 and line[start - 1] != ' ': - start -= 1 - end = col - while end < len(line) and line[end] != ' ': - end += 1 - return line[start:end] - - -def py_examine( - projroot: Path, - filename: Path, - line: int, - column: int, - selection: str | None, - operation: str, -) -> None: - """Given file position info, performs some code inspection.""" - # pylint: disable=too-many-locals - # pylint: disable=cyclic-import - import astroid - import re - from efrotools import code - - # Pull in our pylint plugin which really just adds astroid filters. - # That way our introspection here will see the same thing as pylint's does. - with open(filename, encoding='utf-8') as infile: - fcontents = infile.read() - if '#@' in fcontents: - raise RuntimeError('#@ marker found in file; this breaks examinations.') - flines = fcontents.splitlines() - - if operation == 'pylint_infer': - # See what asteroid can infer about the target symbol. - symbol = ( - selection - if selection is not None - else _py_symbol_at_column(flines[line - 1], column) - ) - - # Insert a line after the provided one which is just the symbol so - # that we can ask for its value alone. - match = re.match(r'\s*', flines[line - 1]) - whitespace = match.group() if match is not None else '' - sline = whitespace + symbol + ' #@' - flines = flines[:line] + [sline] + flines[line:] - node = astroid.extract_node('\n'.join(flines)) - inferred = list(node.infer()) - print(symbol + ':', ', '.join([str(i) for i in inferred])) - elif operation in ('mypy_infer', 'mypy_locals'): - # Ask mypy for the type of the target symbol. - symbol = ( - selection - if selection is not None - else _py_symbol_at_column(flines[line - 1], column) - ) - - # Insert a line after the provided one which is just the symbol so - # that we can ask for its value alone. - match = re.match(r'\s*', flines[line - 1]) - whitespace = match.group() if match is not None else '' - if operation == 'mypy_infer': - sline = whitespace + 'reveal_type(' + symbol + ')' - else: - sline = whitespace + 'reveal_locals()' - flines = flines[:line] + [sline] + flines[line:] - - # Write a temp file and run the check on it. - # Let's use ' flycheck_*' for the name since pipeline scripts - # are already set to ignore those files. - tmppath = Path(filename.parent, 'flycheck_mp_' + filename.name) - with tmppath.open('w', encoding='utf-8') as outfile: - outfile.write('\n'.join(flines)) - try: - code.mypy_files(projroot, [str(tmppath)], check=False) - except Exception as exc: - print('error running mypy:', exc) - tmppath.unlink() - elif operation == 'pylint_node': - flines[line - 1] += ' #@' - node = astroid.extract_node('\n'.join(flines)) - print(node) - elif operation == 'pylint_tree': - flines[line - 1] += ' #@' - node = astroid.extract_node('\n'.join(flines)) - print(node.repr_tree()) - else: - print('unknown operation: ' + operation) diff --git a/tools/efrotools/code.py b/tools/efrotools/code.py index 06ca91dc..e70a6a2e 100644 --- a/tools/efrotools/code.py +++ b/tools/efrotools/code.py @@ -77,7 +77,7 @@ def format_project_cpp_files(projroot: Path, full: bool) -> None: import concurrent.futures from multiprocessing import cpu_count - from efrotools import get_files_hash + from efrotools.util import get_files_hash if os.path.abspath(projroot) != os.getcwd(): raise RuntimeError('We expect to be running from project root.') @@ -137,7 +137,7 @@ def check_cpplint(projroot: Path, full: bool) -> None: from concurrent.futures import ThreadPoolExecutor from multiprocessing import cpu_count - from efrotools import getprojectconfig + from efrotools.project import getprojectconfig from efro.terminal import Clr os.chdir(projroot) @@ -221,7 +221,7 @@ def get_code_filenames(projroot: Path, include_generated: bool) -> list[str]: could cause dirty generated files to not get updated properly when their sources change). """ - from efrotools import getprojectconfig + from efrotools.project import getprojectconfig exts = ('.h', '.c', '.cc', '.cpp', '.cxx', '.m', '.mm') places = getprojectconfig(projroot).get('code_source_dirs', None) @@ -272,7 +272,7 @@ def black_base_args(projroot: Path) -> list[str]: def format_project_python_files(projroot: Path, full: bool) -> None: """Runs formatting on all of our Python code.""" - from efrotools import get_string_hash + from efrotools.util import get_string_hash os.chdir(projroot) cachepath = Path(projroot, '.cache/format_project_python_files') @@ -344,7 +344,7 @@ def _should_include_script(fnamefull: str) -> bool: def get_script_filenames(projroot: Path) -> list[str]: """Return the Python filenames to lint-check or auto-format.""" - from efrotools import getprojectconfig + from efrotools.project import getprojectconfig proot = f'{projroot}/' @@ -392,7 +392,7 @@ def runpylint(projroot: Path, filenames: list[str]) -> None: def pylint(projroot: Path, full: bool, fast: bool) -> None: """Run Pylint on all scripts in our project (with smart dep tracking).""" - from efrotools import get_files_hash + from efrotools.util import get_files_hash from efro.terminal import Clr pylintrc = Path(projroot, '.pylintrc') @@ -568,7 +568,7 @@ def _apply_pylint_run_to_cache( from astroid import modutils - from efrotools import getprojectconfig + from efrotools.project import getprojectconfig # First off, build a map of dirtyfiles to module names # (and the corresponding reverse map). diff --git a/tools/efrotools/efrocache.py b/tools/efrotools/efrocache.py index a716bd24..d2589bdf 100644 --- a/tools/efrotools/efrocache.py +++ b/tools/efrotools/efrocache.py @@ -74,10 +74,9 @@ def get_local_cache_dir() -> str: def get_repository_base_url() -> str: """Return the base repository url (assumes cwd is project root).""" - # from efrotools import getprojectconfig - import efrotools + from efrotools.project import getprojectconfig - pconfig = efrotools.getprojectconfig('.') + pconfig = getprojectconfig('.') name = 'efrocache_repository_url' val = pconfig.get(name) if not isinstance(val, str): diff --git a/tools/efrotools/emacs.py b/tools/efrotools/emacs.py new file mode 100644 index 00000000..ffbc7b81 --- /dev/null +++ b/tools/efrotools/emacs.py @@ -0,0 +1,103 @@ +# Released under the MIT License. See LICENSE for details. +# +"""Stuff intended to be used from emacs""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + pass + + +def _py_symbol_at_column(line: str, col: int) -> str: + start = col + while start > 0 and line[start - 1] != ' ': + start -= 1 + end = col + while end < len(line) and line[end] != ' ': + end += 1 + return line[start:end] + + +def py_examine( + projroot: Path, + filename: Path, + line: int, + column: int, + selection: str | None, + operation: str, +) -> None: + """Given file position info, performs some code inspection.""" + # pylint: disable=too-many-locals + # pylint: disable=cyclic-import + import astroid + import re + from efrotools import code + + # Pull in our pylint plugin which really just adds astroid filters. + # That way our introspection here will see the same thing as pylint's does. + with open(filename, encoding='utf-8') as infile: + fcontents = infile.read() + if '#@' in fcontents: + raise RuntimeError('#@ marker found in file; this breaks examinations.') + flines = fcontents.splitlines() + + if operation == 'pylint_infer': + # See what asteroid can infer about the target symbol. + symbol = ( + selection + if selection is not None + else _py_symbol_at_column(flines[line - 1], column) + ) + + # Insert a line after the provided one which is just the symbol so + # that we can ask for its value alone. + match = re.match(r'\s*', flines[line - 1]) + whitespace = match.group() if match is not None else '' + sline = whitespace + symbol + ' #@' + flines = flines[:line] + [sline] + flines[line:] + node = astroid.extract_node('\n'.join(flines)) + inferred = list(node.infer()) + print(symbol + ':', ', '.join([str(i) for i in inferred])) + elif operation in ('mypy_infer', 'mypy_locals'): + # Ask mypy for the type of the target symbol. + symbol = ( + selection + if selection is not None + else _py_symbol_at_column(flines[line - 1], column) + ) + + # Insert a line after the provided one which is just the symbol so + # that we can ask for its value alone. + match = re.match(r'\s*', flines[line - 1]) + whitespace = match.group() if match is not None else '' + if operation == 'mypy_infer': + sline = whitespace + 'reveal_type(' + symbol + ')' + else: + sline = whitespace + 'reveal_locals()' + flines = flines[:line] + [sline] + flines[line:] + + # Write a temp file and run the check on it. + # Let's use ' flycheck_*' for the name since pipeline scripts + # are already set to ignore those files. + tmppath = Path(filename.parent, 'flycheck_mp_' + filename.name) + with tmppath.open('w', encoding='utf-8') as outfile: + outfile.write('\n'.join(flines)) + try: + code.mypy_files(projroot, [str(tmppath)], check=False) + except Exception as exc: + print('error running mypy:', exc) + tmppath.unlink() + elif operation == 'pylint_node': + flines[line - 1] += ' #@' + node = astroid.extract_node('\n'.join(flines)) + print(node) + elif operation == 'pylint_tree': + flines[line - 1] += ' #@' + node = astroid.extract_node('\n'.join(flines)) + print(node.repr_tree()) + else: + print('unknown operation: ' + operation) diff --git a/tools/efrotools/filecache.py b/tools/efrotools/filecache.py index 711fb637..d5d9dc22 100644 --- a/tools/efrotools/filecache.py +++ b/tools/efrotools/filecache.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING # pylint: disable=useless-suppression # pylint: disable=wrong-import-order from efro.terminal import Clr -from efrotools import get_files_hash +from efrotools.util import get_files_hash # pylint: enable=wrong-import-order # pylint: enable=useless-suppression diff --git a/tools/efrotools/genwrapper.py b/tools/efrotools/genwrapper.py index 36921296..92dddb16 100644 --- a/tools/efrotools/genwrapper.py +++ b/tools/efrotools/genwrapper.py @@ -15,8 +15,8 @@ if __name__ == '__main__': if len(sys.argv) not in (3, 4): raise RuntimeError('Expected 2 args') - from efrotools import getprojectconfig from efrotools.project import ( + getprojectconfig, get_public_legal_notice, get_non_public_legal_notice, ) diff --git a/tools/efrotools/ios.py b/tools/efrotools/ios.py index db10b8c5..cff28adf 100644 --- a/tools/efrotools/ios.py +++ b/tools/efrotools/ios.py @@ -9,7 +9,7 @@ import subprocess import sys from dataclasses import dataclass -from efrotools import getprojectconfig, getlocalconfig +from efrotools.project import getprojectconfig, getlocalconfig MODES = { 'debug': {'configuration': 'Debug'}, diff --git a/tools/efrotools/lazybuild.py b/tools/efrotools/lazybuild.py index fbdc8f49..37242e67 100644 --- a/tools/efrotools/lazybuild.py +++ b/tools/efrotools/lazybuild.py @@ -16,7 +16,7 @@ from typing import TYPE_CHECKING # pylint: disable=wrong-import-order from efro.terminal import Clr from efrotools.buildlock import BuildLock -from efrotools import get_string_hash +from efrotools.util import get_string_hash # pylint: enable=wrong-import-order # pylint: enable=useless-suppression diff --git a/tools/efrotools/openalbuild.py b/tools/efrotools/openalbuild.py index 19518381..2215cb03 100644 --- a/tools/efrotools/openalbuild.py +++ b/tools/efrotools/openalbuild.py @@ -33,7 +33,7 @@ def build_openal(arch: str, mode: str) -> None: """Do the thing.""" # pylint: disable=too-many-statements # pylint: disable=too-many-locals - from efrotools import replace_exact + from efrotools.util import replace_exact if arch not in ARCHS: raise CleanError(f"Invalid arch '{arch}'.") diff --git a/tools/efrotools/pcommands.py b/tools/efrotools/pcommands.py index a5cbee2b..47fe9b8c 100644 --- a/tools/efrotools/pcommands.py +++ b/tools/efrotools/pcommands.py @@ -477,7 +477,7 @@ def sync_all() -> None: def sync() -> None: """Runs standard syncs between this project and others.""" - from efrotools import getprojectconfig + from efrotools.project import getprojectconfig from efrotools.sync import Mode, SyncItem, run_standard_syncs pcommand.disallow_in_batch() @@ -625,7 +625,7 @@ def pytest() -> None: import os import platform import subprocess - from efrotools import getprojectconfig + from efrotools.project import getprojectconfig from efro.error import CleanError pcommand.disallow_in_batch() diff --git a/tools/efrotools/pcommands2.py b/tools/efrotools/pcommands2.py index 76beef8a..10648db0 100644 --- a/tools/efrotools/pcommands2.py +++ b/tools/efrotools/pcommands2.py @@ -122,7 +122,7 @@ def batchserver() -> None: """Run a server for handling pcommands.""" from efro.error import CleanError - from efrotools import extract_arg + from efrotools.util import extract_arg import efrotools.pcommandbatch as pcb pcommand.disallow_in_batch() diff --git a/tools/efrotools/project.py b/tools/efrotools/project.py index b2587c8f..908f9b8b 100644 --- a/tools/efrotools/project.py +++ b/tools/efrotools/project.py @@ -4,10 +4,17 @@ from __future__ import annotations +import os +import json +from pathlib import Path from typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Literal + from typing import Literal, Any + +# Cache these since we may repeatedly fetch these in batch mode. +_g_project_configs: dict[str, dict[str, Any]] = {} +_g_local_configs: dict[str, dict[str, Any]] = {} def get_public_legal_notice( @@ -40,3 +47,51 @@ def get_non_public_legal_notice_prev() -> str: """Allows us to auto-update.""" # TODO: Move this to project config or somewhere not hard-coded. return 'Copyright (c) 2011-2023 Eric Froemling' + + +def getlocalconfig(projroot: Path | str) -> dict[str, Any]: + """Return a project's localconfig contents (or default if missing).""" + projrootstr = str(projroot) + if projrootstr not in _g_local_configs: + localconfig: dict[str, Any] + + # Allow overriding path via env var. + path = os.environ.get('EFRO_LOCALCONFIG_PATH') + if path is None: + path = 'config/localconfig.json' + + try: + with open(Path(projroot, path), encoding='utf-8') as infile: + localconfig = json.loads(infile.read()) + except FileNotFoundError: + localconfig = {} + _g_local_configs[projrootstr] = localconfig + + return _g_local_configs[projrootstr] + + +def getprojectconfig(projroot: Path | str) -> dict[str, Any]: + """Return a project's projectconfig contents (or default if missing).""" + projrootstr = str(projroot) + if projrootstr not in _g_project_configs: + config: dict[str, Any] + try: + with open( + Path(projroot, 'config/projectconfig.json'), encoding='utf-8' + ) as infile: + config = json.loads(infile.read()) + except FileNotFoundError: + config = {} + _g_project_configs[projrootstr] = config + return _g_project_configs[projrootstr] + + +def setprojectconfig(projroot: Path | str, config: dict[str, Any]) -> None: + """Set the project config contents.""" + projrootstr = str(projroot) + _g_project_configs[projrootstr] = config + os.makedirs(Path(projroot, 'config'), exist_ok=True) + with Path(projroot, 'config/projectconfig.json').open( + 'w', encoding='utf-8' + ) as outfile: + outfile.write(json.dumps(config, indent=2)) diff --git a/tools/efrotools/pybuild.py b/tools/efrotools/pybuild.py index a06ff671..17dc2958 100644 --- a/tools/efrotools/pybuild.py +++ b/tools/efrotools/pybuild.py @@ -10,7 +10,7 @@ import subprocess from enum import Enum from dataclasses import dataclass -from efrotools import readfile, writefile, replace_exact +from efrotools.util import readfile, writefile, replace_exact # Python version we build here (not necessarily same as we use in repo). PY_VER_ANDROID = '3.12' diff --git a/tools/efrotools/sync.py b/tools/efrotools/sync.py index 71cf7ddd..22fcc47b 100644 --- a/tools/efrotools/sync.py +++ b/tools/efrotools/sync.py @@ -71,7 +71,7 @@ def run_standard_syncs( a src subpath, and optionally a dst subpath (src will be used by default). """ # pylint: disable=too-many-locals - from efrotools import getlocalconfig + from efrotools.project import getlocalconfig localconfig = getlocalconfig(projectroot) total_count = 0 diff --git a/tools/efrotools/toolconfig.py b/tools/efrotools/toolconfig.py index 0610ad77..648202bb 100644 --- a/tools/efrotools/toolconfig.py +++ b/tools/efrotools/toolconfig.py @@ -68,7 +68,7 @@ def _filter_tool_config(projroot: Path, cfg: str) -> str: # pylint: disable=too-many-locals import textwrap - from efrotools import getprojectconfig + from efrotools.project import getprojectconfig from efrotools.pyver import PYVER # Emacs dir-locals defaults. Note that these contain other diff --git a/tools/efrotools/util.py b/tools/efrotools/util.py new file mode 100644 index 00000000..b0da6206 --- /dev/null +++ b/tools/efrotools/util.py @@ -0,0 +1,191 @@ +# Released under the MIT License. See LICENSE for details. +# +"""Misc util calls/etc. + +Ideally the stuff in here should migrate to more descriptive module names. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, overload + + +if TYPE_CHECKING: + from typing import Sequence, Literal + from pathlib import Path + + +def explicit_bool(value: bool) -> bool: + """Simply return input value; can avoid unreachable-code type warnings.""" + return value + + +def extract_flag(args: list[str], name: str) -> bool: + """Given a list of args and a flag name, returns whether it is present. + + The arg flag, if present, is removed from the arg list. + """ + from efro.error import CleanError + + count = args.count(name) + if count > 1: + raise CleanError(f'Flag {name} passed multiple times.') + if not count: + return False + args.remove(name) + return True + + +@overload +def extract_arg( + args: list[str], name: str, required: Literal[False] = False +) -> str | None: ... + + +@overload +def extract_arg(args: list[str], name: str, required: Literal[True]) -> str: ... + + +def extract_arg( + args: list[str], name: str, required: bool = False +) -> str | None: + """Given a list of args and an arg name, returns a value. + + The arg flag and value are removed from the arg list. + raises CleanErrors on any problems. + """ + from efro.error import CleanError + + count = args.count(name) + if not count: + if required: + raise CleanError(f'Required argument {name} not passed.') + return None + + if count > 1: + raise CleanError(f'Arg {name} passed multiple times.') + + argindex = args.index(name) + if argindex + 1 >= len(args): + raise CleanError(f'No value passed after {name} arg.') + + val = args[argindex + 1] + del args[argindex : argindex + 2] + + return val + + +def replace_section( + text: str, + begin_marker: str, + end_marker: str, + replace_text: str = '', + keep_markers: bool = False, + error_if_missing: bool = True, +) -> str: + """Replace all text between two marker strings (including the markers).""" + if begin_marker not in text: + if error_if_missing: + raise RuntimeError(f"Marker not found in text: '{begin_marker}'.") + return text + splits = text.split(begin_marker) + if len(splits) != 2: + raise RuntimeError( + f"Expected one marker '{begin_marker}'" + f'; found {text.count(begin_marker)}.' + ) + before_begin, after_begin = splits + splits = after_begin.split(end_marker) + if len(splits) != 2: + raise RuntimeError( + f"Expected one marker '{end_marker}'" + f'; found {text.count(end_marker)}.' + ) + _before_end, after_end = splits + if keep_markers: + replace_text = f'{begin_marker}{replace_text}{end_marker}' + return f'{before_begin}{replace_text}{after_end}' + + +def readfile(path: str | Path) -> str: + """Read a utf-8 text file into a string.""" + with open(path, encoding='utf-8') as infile: + return infile.read() + + +def writefile(path: str | Path, txt: str) -> None: + """Write a string to a utf-8 text file.""" + with open(path, 'w', encoding='utf-8') as outfile: + outfile.write(txt) + + +def replace_exact( + opstr: str, old: str, new: str, count: int = 1, label: str | None = None +) -> str: + """Replace text ensuring that exactly x occurrences are replaced. + + Useful when filtering data in some predefined way to ensure the original + has not changed. + """ + found = opstr.count(old) + label_str = f' in {label}' if label is not None else '' + if found != count: + raise RuntimeError( + f'Expected {count} string occurrence(s){label_str};' + f' found {found}. String: {repr(old)}' + ) + return opstr.replace(old, new) + + +def get_files_hash( + filenames: Sequence[str | Path], + extrahash: str = '', + int_only: bool = False, + hashtype: Literal['md5', 'sha256'] = 'md5', +) -> str: + """Return a hash for the given files.""" + import hashlib + + if not isinstance(filenames, list): + raise RuntimeError(f'Expected a list; got a {type(filenames)}.') + if TYPE_CHECKING: + # Help Mypy infer the right type for this. + hashobj = hashlib.md5() + else: + hashobj = getattr(hashlib, hashtype)() + for fname in filenames: + with open(fname, 'rb') as infile: + while True: + data = infile.read(2**20) + if not data: + break + hashobj.update(data) + hashobj.update(extrahash.encode()) + + if int_only: + return str(int.from_bytes(hashobj.digest(), byteorder='big')) + + return hashobj.hexdigest() + + +def get_string_hash( + value: str, + int_only: bool = False, + hashtype: Literal['md5', 'sha256'] = 'md5', +) -> str: + """Return a hash for the given files.""" + import hashlib + + if not isinstance(value, str): + raise TypeError('Expected a str.') + if TYPE_CHECKING: + # Help Mypy infer the right type for this. + hashobj = hashlib.md5() + else: + hashobj = getattr(hashlib, hashtype)() + hashobj.update(value.encode()) + + if int_only: + return str(int.from_bytes(hashobj.digest(), byteorder='big')) + + return hashobj.hexdigest() diff --git a/tools/efrotools/xcodebuild.py b/tools/efrotools/xcodebuild.py index c221ef7b..e717076d 100644 --- a/tools/efrotools/xcodebuild.py +++ b/tools/efrotools/xcodebuild.py @@ -22,7 +22,7 @@ from filelock import FileLock from efro.terminal import Clr from efro.error import CleanError from efro.dataclassio import ioprepped, dataclass_from_dict -from efrotools import getlocalconfig # pylint: disable=wrong-import-order +from efrotools.project import getlocalconfig # pylint: disable=C0411 if TYPE_CHECKING: from typing import Any diff --git a/tools/spinoff b/tools/spinoff index 15b9f8fe..63a93ec8 100755 --- a/tools/spinoff +++ b/tools/spinoff @@ -10,36 +10,50 @@ import sys import subprocess if __name__ == '__main__': - from batools.spinoff import spinoff_main - # Our initial invocation actually just sets up the env for our - # *real* invocation (so we can run under our desired venv/etc.) - if os.environ.get('BA_SPINOFF_HAVE_ENV') != '1': + # The initial invocation of this script actually just sets the stage + # for the *real* invocation of this script, which always happens + # from the fully-inited virtual env of the source project. This way + # all modules used by the spinoff system are in place and there's no + # abiguity where we could be loading Python stuff from the dst + # project while we're in the process of modifying it. + if 'BA_SPINOFF_SRC_ROOT' not in os.environ: - # Our shebang line gives us a generic 'pythonX.Y' environment, but - # we actually want to run under the virtual-env of the source - # project so we have all the pip stuff we expect. So if we are - # getting invoked via a symlink we assume it points to the source - # project, and if not then we assume we are the source project. + # Calc absolute paths for our source (and possibly dst) + # projects. If we are getting invoked via a symlink, what it + # points to is src and we are dst. Otherwise we are src and + # there is no dst. + dst_proj_root: str | None if os.path.islink(sys.argv[0]): src_spinoff_path = os.path.realpath(sys.argv[0]) + dst_proj_root = os.path.abspath( + os.path.join(os.path.dirname(sys.argv[0]), '..') + ) + else: src_spinoff_path = sys.argv[0] + dst_proj_root = None # pylint: disable=invalid-name src_proj_root = os.path.abspath( os.path.join(os.path.dirname(src_spinoff_path), '..') ) src_proj_python = os.path.join(src_proj_root, '.venv/bin/python3.12') - cmd = [src_proj_python, sys.argv[0]] + sys.argv[1:] + cmd = [src_proj_python, 'tools/spinoff'] + sys.argv[1:] + + env = dict(os.environ, BA_SPINOFF_SRC_ROOT=src_proj_root) + if dst_proj_root is not None: + env['BA_SPINOFF_DST_ROOT'] = dst_proj_root # Make sure the src project is properly bootstrapped. subprocess.run(['make', 'prereqs'], check=True, cwd=src_proj_root) - # Finally, run for realz. - subprocess.run( - cmd, check=True, env=dict(os.environ, BA_SPINOFF_HAVE_ENV='1') - ) + # Finally, run for realz (from src proj dir). + result = subprocess.run(cmd, check=False, env=env, cwd=src_proj_root) + sys.exit(result.returncode) else: + from batools.spinoff import spinoff_main + + # Ok; we're a real invocation. Do our thing. spinoff_main()