diff --git a/.efrocachemap b/.efrocachemap index 12c18823..fc567c7c 100644 --- a/.efrocachemap +++ b/.efrocachemap @@ -3966,50 +3966,50 @@ "assets/src/ba_data/python/ba/_generated/__init__.py": "https://files.ballistica.net/cache/ba1/ee/e8/cad05aa531c7faf7ff7b96db7f6e", "assets/src/ba_data/python/ba/_generated/enums.py": "https://files.ballistica.net/cache/ba1/72/82/86956fae909ac2fe2a1abd84a361", "ballisticacore-windows/Generic/BallisticaCore.ico": "https://files.ballistica.net/cache/ba1/89/c0/e32c7d2a35dc9aef57cc73b0911a", - "build/prefab/full/linux_arm64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/0d/3f/92b3e229a7211ffeb5cd6b777364", + "build/prefab/full/linux_arm64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/88/d6/2de73f757e3bdbfeaec03f563c8a", "build/prefab/full/linux_arm64_gui/release/ballisticacore": "https://files.ballistica.net/cache/ba1/74/1d/fc9e33e565475daaac80da5252f0", - "build/prefab/full/linux_arm64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/40/6b/8200d88decb9480bf873df2daefe", - "build/prefab/full/linux_arm64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/86/fc/149d346aa7fb17c9d176d5c71b5a", - "build/prefab/full/linux_x86_64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/89/9e/8411e30418cac89810c8bd5e422a", + "build/prefab/full/linux_arm64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/09/93/182c306fdfe104a925794984a763", + "build/prefab/full/linux_arm64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/c1/43/f37d6f9200d7c72d06ad158e6a6b", + "build/prefab/full/linux_x86_64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/db/dd/51cd186c11c6399c8e39c306652b", "build/prefab/full/linux_x86_64_gui/release/ballisticacore": "https://files.ballistica.net/cache/ba1/9c/7b/ac1a200be0f37078af0991faca3b", - "build/prefab/full/linux_x86_64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/4b/dd/52fdcf376bf5a577ec4548a34c27", - "build/prefab/full/linux_x86_64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/2c/3c/f386b9c98d2e3b7cc69c5aa5bac0", - "build/prefab/full/mac_arm64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/14/8e/e2f299fde04604aa7a727b6d7526", - "build/prefab/full/mac_arm64_gui/release/ballisticacore": "https://files.ballistica.net/cache/ba1/58/6d/afb3a95d0774e7ca227f77202866", - "build/prefab/full/mac_arm64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/d2/ee/e4bcb96a463516e1068714c39c4d", - "build/prefab/full/mac_arm64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/28/a9/ba7d4be38d707695d942173c2380", - "build/prefab/full/mac_x86_64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/1a/3b/f9fe01cf3a776cceecd078c99f92", - "build/prefab/full/mac_x86_64_gui/release/ballisticacore": "https://files.ballistica.net/cache/ba1/42/53/07d28b98ff2ffc857438d07268ea", - "build/prefab/full/mac_x86_64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/e2/2c/4ec233e815eadc0b427391fe3867", - "build/prefab/full/mac_x86_64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/a4/dc/a8f17a3c35c6ead2580ab32157c0", - "build/prefab/full/windows_x86_gui/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/49/21/b2ca5e3af03446536329a8307ebd", - "build/prefab/full/windows_x86_gui/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/d0/fe/5ee21da4a647e55e3b3b57f7af66", - "build/prefab/full/windows_x86_server/debug/dist/BallisticaCoreHeadless.exe": "https://files.ballistica.net/cache/ba1/91/0b/d758e25cd70756a0654a368f3c6c", - "build/prefab/full/windows_x86_server/release/dist/BallisticaCoreHeadless.exe": "https://files.ballistica.net/cache/ba1/51/d7/05324adaa59b8f5be699a34d9df3", - "build/prefab/lib/linux_arm64_gui/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/00/bc/3cbd2cc3820bb81bbd972a7ceb5a", - "build/prefab/lib/linux_arm64_gui/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/d3/5c/969e1b4c0958ae64db139742be65", - "build/prefab/lib/linux_arm64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/29/ce/80de999ebf7f31e4711c4fbcd567", - "build/prefab/lib/linux_arm64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/65/e6/694cceb311b14db0e7e2a7d222e3", - "build/prefab/lib/linux_x86_64_gui/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/ff/5f/63148f1884e1d54a18f6628cf6d7", - "build/prefab/lib/linux_x86_64_gui/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/16/c7/eef151836539d51fe613de96a628", - "build/prefab/lib/linux_x86_64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/41/62/a68605fc59816419ef378cf111df", - "build/prefab/lib/linux_x86_64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/65/9a/fb4eead17d1f0443868ae4a3dcba", - "build/prefab/lib/mac_arm64_gui/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/2a/30/2e7ca4dbacd3067e8194180b4a2d", - "build/prefab/lib/mac_arm64_gui/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/94/65/197a50a86aacc67f1a2e274febb8", - "build/prefab/lib/mac_arm64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/ca/55/af7c67cb2fc965a260b85c55a26a", - "build/prefab/lib/mac_arm64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/11/ef/73f5bd0b9fbab59ebf66f3ced62b", - "build/prefab/lib/mac_x86_64_gui/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/01/85/7a8bc2e018eb9c74c456a94c53c0", - "build/prefab/lib/mac_x86_64_gui/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/ab/6e/a567ec80b534bc904f5abca70191", - "build/prefab/lib/mac_x86_64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/1f/88/26cc23cc420b9ee4484604d83f3b", - "build/prefab/lib/mac_x86_64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/02/62/6c4447768c7dc1b347d74f8bbb64", - "build/prefab/lib/windows/Debug_Win32/BallisticaCoreGenericInternal.lib": "https://files.ballistica.net/cache/ba1/0d/49/4e44dc53764b5024d38c5754ff04", - "build/prefab/lib/windows/Debug_Win32/BallisticaCoreGenericInternal.pdb": "https://files.ballistica.net/cache/ba1/51/a4/836e5731f71d26ee8c768ac40dc9", - "build/prefab/lib/windows/Debug_Win32/BallisticaCoreHeadlessInternal.lib": "https://files.ballistica.net/cache/ba1/41/ed/0d46c0017ebe8cfd9b335e03775f", - "build/prefab/lib/windows/Debug_Win32/BallisticaCoreHeadlessInternal.pdb": "https://files.ballistica.net/cache/ba1/bd/38/2ebf14bb77f34100636c7f98b7ed", - "build/prefab/lib/windows/Release_Win32/BallisticaCoreGenericInternal.lib": "https://files.ballistica.net/cache/ba1/19/0e/aecf6e368c3142bab74abe064c59", - "build/prefab/lib/windows/Release_Win32/BallisticaCoreGenericInternal.pdb": "https://files.ballistica.net/cache/ba1/94/9c/ca0b324101aca56c4b22a0134bfd", - "build/prefab/lib/windows/Release_Win32/BallisticaCoreHeadlessInternal.lib": "https://files.ballistica.net/cache/ba1/9c/6f/0ddd22e0c4a14e1cce1d8371e7b3", - "build/prefab/lib/windows/Release_Win32/BallisticaCoreHeadlessInternal.pdb": "https://files.ballistica.net/cache/ba1/01/ea/cece952515d0ba6811fd03f7b7d7", + "build/prefab/full/linux_x86_64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/94/5f/f596a1c6ec46b059997038b2ad98", + "build/prefab/full/linux_x86_64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/36/4b/54a3508504842e449bc4632facc7", + "build/prefab/full/mac_arm64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/ad/6a/39489935b6d36077d0180d365f65", + "build/prefab/full/mac_arm64_gui/release/ballisticacore": "https://files.ballistica.net/cache/ba1/1c/30/37cb8de37563ece076f3df06407e", + "build/prefab/full/mac_arm64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/61/d5/8d839aa2b10e2c267c661f22f131", + "build/prefab/full/mac_arm64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/92/db/36e17bb4062382cd0d07818415d1", + "build/prefab/full/mac_x86_64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/f1/39/d44269765fa93903c0b8896b4f62", + "build/prefab/full/mac_x86_64_gui/release/ballisticacore": "https://files.ballistica.net/cache/ba1/f2/e3/64b2d15ae4c8f2d060f0ac782512", + "build/prefab/full/mac_x86_64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/48/10/c493cee7128f6c5caa0c450df55d", + "build/prefab/full/mac_x86_64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/68/2c/c65363cd1d7dd78f425fd9586258", + "build/prefab/full/windows_x86_gui/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/b5/7c/ccec2d40ddafa91261b086d1173c", + "build/prefab/full/windows_x86_gui/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/78/f4/3356065ce9b87d87fbd29acd8cac", + "build/prefab/full/windows_x86_server/debug/dist/BallisticaCoreHeadless.exe": "https://files.ballistica.net/cache/ba1/b7/9b/e3f766f96f89650642a5ddc6a4ea", + "build/prefab/full/windows_x86_server/release/dist/BallisticaCoreHeadless.exe": "https://files.ballistica.net/cache/ba1/20/13/4b60da93efe23ac4c7cf9a350bc7", + "build/prefab/lib/linux_arm64_gui/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/5f/7c/427e8a1af1c95bf40211adb1a10e", + "build/prefab/lib/linux_arm64_gui/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/5e/fe/0366d7ace626419f14c2bd80e266", + "build/prefab/lib/linux_arm64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/9b/2c/7be9dce2d9aff8cf04508d07dff0", + "build/prefab/lib/linux_arm64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/0e/c3/b9d0325bee5ee116a1311a87dd11", + "build/prefab/lib/linux_x86_64_gui/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/8b/ff/1345a0a9d07ea8b7164ec2a97234", + "build/prefab/lib/linux_x86_64_gui/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/d7/2c/9fa96710e25f6242c42ff3efd309", + "build/prefab/lib/linux_x86_64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/89/45/43b51cddf1b4655d31018158b344", + "build/prefab/lib/linux_x86_64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/8c/6d/c3f165109101ae65f3ecbc556492", + "build/prefab/lib/mac_arm64_gui/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/44/f1/1cfd8a4bc54d87f86bbfdd9fa439", + "build/prefab/lib/mac_arm64_gui/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/d9/c2/15a504dbea88fe4eda1c57f67c76", + "build/prefab/lib/mac_arm64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/3f/1e/3e78487f4d70a2c5974ec0198fb1", + "build/prefab/lib/mac_arm64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/18/4a/48fd42b279a165b41c43c7973f5b", + "build/prefab/lib/mac_x86_64_gui/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/ae/61/8f2ab7f636f09ff8cabf7e5d8d37", + "build/prefab/lib/mac_x86_64_gui/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/66/bb/3f0591d11d845c285ceb0e322121", + "build/prefab/lib/mac_x86_64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/81/a4/21f0084dde2fc524b4683b5efed0", + "build/prefab/lib/mac_x86_64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/9f/03/dbe0091c812e01eb84b66de55b36", + "build/prefab/lib/windows/Debug_Win32/BallisticaCoreGenericInternal.lib": "https://files.ballistica.net/cache/ba1/6a/c2/6784b83d7cc684260cf162fbee4c", + "build/prefab/lib/windows/Debug_Win32/BallisticaCoreGenericInternal.pdb": "https://files.ballistica.net/cache/ba1/28/1e/781487d4d6464f27a43b3bb1c2f4", + "build/prefab/lib/windows/Debug_Win32/BallisticaCoreHeadlessInternal.lib": "https://files.ballistica.net/cache/ba1/38/1a/a53ba81dd431860600bc5c21e0fa", + "build/prefab/lib/windows/Debug_Win32/BallisticaCoreHeadlessInternal.pdb": "https://files.ballistica.net/cache/ba1/f8/6f/0d0cf942205991c889722a116aab", + "build/prefab/lib/windows/Release_Win32/BallisticaCoreGenericInternal.lib": "https://files.ballistica.net/cache/ba1/65/c8/47bc70c1b14593f5928d5d2c2b90", + "build/prefab/lib/windows/Release_Win32/BallisticaCoreGenericInternal.pdb": "https://files.ballistica.net/cache/ba1/ef/01/0bd5c7939ae809ddf0334927d24e", + "build/prefab/lib/windows/Release_Win32/BallisticaCoreHeadlessInternal.lib": "https://files.ballistica.net/cache/ba1/90/dc/491604fafce64d74578ab8f49f7b", + "build/prefab/lib/windows/Release_Win32/BallisticaCoreHeadlessInternal.pdb": "https://files.ballistica.net/cache/ba1/12/2d/96d202ba712d4efdcb61591e6e40", "src/ballistica/generated/python_embedded/binding.inc": "https://files.ballistica.net/cache/ba1/f2/6c/5a0a4695dcc2a11e7941b8777e80", "src/ballistica/generated/python_embedded/bootstrap.inc": "https://files.ballistica.net/cache/ba1/65/ac/d5c4162a71028c1bfa73ebc1f881" } \ No newline at end of file diff --git a/.idea/dictionaries/ericf.xml b/.idea/dictionaries/ericf.xml index 5b8f0b3e..eec6298a 100644 --- a/.idea/dictionaries/ericf.xml +++ b/.idea/dictionaries/ericf.xml @@ -1545,6 +1545,7 @@ nowtickets npos nprocessors + nsworkspace ntpath ntriple nturl @@ -1905,6 +1906,7 @@ realpath realsies recache + recursed recv redist redistributables diff --git a/ballisticacore-cmake/.idea/dictionaries/ericf.xml b/ballisticacore-cmake/.idea/dictionaries/ericf.xml index a145da7b..8da45e13 100644 --- a/ballisticacore-cmake/.idea/dictionaries/ericf.xml +++ b/ballisticacore-cmake/.idea/dictionaries/ericf.xml @@ -726,6 +726,7 @@ nowtickets nptr nsize + nsworkspace ntoa ntohl numargs @@ -894,6 +895,7 @@ reallocations realtimers recalc + recursed recv recvfrom redundants diff --git a/config/config.json b/config/config.json index 4ec94dd9..0a17a714 100644 --- a/config/config.json +++ b/config/config.json @@ -32,7 +32,8 @@ "typing_extensions", "cpplint", "ansiwrap", - "filelock" + "filelock", + "Cocoa" ], "python_paths": [ "assets/src/ba_data/python", diff --git a/config/toolconfigsrc/mypy.ini b/config/toolconfigsrc/mypy.ini index f810dd0b..1e385f3d 100644 --- a/config/toolconfigsrc/mypy.ini +++ b/config/toolconfigsrc/mypy.ini @@ -16,6 +16,9 @@ no_implicit_reexport = False [mypy-ba.deprecated] no_implicit_reexport = False +[mypy-Cocoa.*] +ignore_missing_imports = True + [mypy-pylint.*] ignore_missing_imports = True diff --git a/src/ballistica/ballistica.cc b/src/ballistica/ballistica.cc index 4bcfa1ee..f215e8ab 100644 --- a/src/ballistica/ballistica.cc +++ b/src/ballistica/ballistica.cc @@ -21,7 +21,7 @@ namespace ballistica { // These are set automatically via script; don't modify them here. -const int kAppBuildNumber = 20406; +const int kAppBuildNumber = 20408; const char* kAppVersion = "1.6.6"; // Our standalone globals. diff --git a/tools/bacloud b/tools/bacloud index 39f17c76..5fd33c2f 100755 --- a/tools/bacloud +++ b/tools/bacloud @@ -30,8 +30,7 @@ VERSION = 1 TOOL_NAME = 'bacloud' -# Set BACLOUD_SERVER env var to LOCAL to talk to a locally-run master-server. -# Set it to TEST to talk to the 'test' app-engine service. +# Server we talk to (can override via env var). MASTER_SERVER_URL = os.getenv('BACLOUD_SERVER_URL', 'https://bamaster.appspot.com') @@ -55,11 +54,11 @@ class Response: with subsequent commands. logout: If True, any existing client-side token should be discarded. dir_manifest: If present, client should generate a manifest of this dir. - It should be added to endcommand args as 'manifest'. + It should be added to end_command args as 'manifest'. uploads: If present, client should upload the requested files (arg1) individually to a server command (arg2) with provided args (arg3). uploads_inline: If present, a list of pathnames that should be base64 - gzipped and uploaded to an 'uploads_inline' dict in endcommand args. + gzipped and uploaded to an 'uploads_inline' dict in end_command args. This should be limited to relatively small files. downloads_inline: If present, pathnames mapped to base64 gzipped data to be written to the client. This should only be used for relatively @@ -69,7 +68,7 @@ class Response: removed. open_url: If present, url to display to the user. input_prompt: If present, a line of input is read and placed into - endcommand args as 'input'. The first value is the prompt printed + end_command args as 'input'. The first value is the prompt printed before reading and the second is whether it should be read as a password (without echoing to the terminal). end_message: If present, a message that should be printed after all other diff --git a/tools/batools/build.py b/tools/batools/build.py index 99401f5d..2d61119b 100644 --- a/tools/batools/build.py +++ b/tools/batools/build.py @@ -40,7 +40,7 @@ PIP_REQUIREMENTS = [ PipRequirement(modulename='ansiwrap'), PipRequirement(modulename='yaml', pipname='PyYAML'), PipRequirement(modulename='requests'), - PipRequirement(pipname='typing-extensions', minversion=[3, 10, 0, 0]), + PipRequirement(pipname='typing_extensions', minversion=[4, 0, 0]), PipRequirement(pipname='types-filelock', minversion=[0, 1, 5]), PipRequirement(pipname='types-requests', minversion=[2, 25, 6]), PipRequirement(pipname='types-pytz', minversion=[2021, 1, 2]), diff --git a/tools/efrotools/filecommand.py b/tools/efrotools/filecommand.py new file mode 100644 index 00000000..e4e1bcf5 --- /dev/null +++ b/tools/efrotools/filecommand.py @@ -0,0 +1,142 @@ +# Released under the MIT License. See LICENSE for details. +# +"""Operate on large sets of files efficiently.""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING +from threading import Condition, Thread +import os + +if TYPE_CHECKING: + from typing import Generator, Optional, Callable + + +class _FileBatchesRun: + + def __init__(self, + paths: list[str], + batch_size: int, + file_filter: Optional[Callable[[str], bool]], + include_mac_packages: bool = False) -> None: + self.condition = Condition() + self.paths = paths + self.batches: list[list[str]] = [] + self.batch_size = batch_size + self.done = False + self.errored = False + self.file_filter = file_filter + self.batch_buffer_size = 5 + self._pending_batch: list[str] = [] + self._include_mac_packages = include_mac_packages + + if self._include_mac_packages: + # pylint: disable=no-name-in-module + from Cocoa import NSWorkspace + self._shared_nsworkspace = NSWorkspace.sharedWorkspace() + else: + self._shared_nsworkspace = None + + def _submit_pending_batch(self) -> None: + assert self._pending_batch + + # Wait until there's room on the list (or we've been marked done), + # stuff our new results in, and inform any listeners that it has + # changed. + with self.condition: + self.condition.wait_for(lambda: len(self.batches) < self. + batch_buffer_size or self.done) + self.batches.append(self._pending_batch) + self._pending_batch = [] + self.condition.notify() + + def _possibly_add_to_pending_batch(self, path: str) -> None: + try: + if self.file_filter is None or self.file_filter(path): + self._pending_batch.append(path) + if len(self._pending_batch) >= self.batch_size: + self._submit_pending_batch() + except Exception: + # FIXME: we should translate this into failing overall... + logging.exception('Error in file_filter') + + def bg_thread(self) -> None: + """Add batches in the bg thread.""" + # pylint: disable=too-many-nested-blocks + + # Build batches and push them when they're big enough. + for path in self.paths: + if os.path.isfile(path): + self._possibly_add_to_pending_batch(path) + elif os.path.isdir(path): + + # From os.walk docs: we can prune dirs in-place when + # running in top-down mode. We can use this to skip + # diving into mac packages. + for root, dirs, fnames in os.walk(path, topdown=True): + + # If we find dirs that are actually mac packages, pull + # them out of the dir list we'll dive into and pass + # them directly to our batch for processing. + if self._include_mac_packages: + for dirname in list(dirs): + fullpath = os.path.join(root, dirname) + if (self._shared_nsworkspace.isFilePackageAtPath_( + fullpath)): + dirs.remove(dirname) + self._possibly_add_to_pending_batch(fullpath) + + for fname in fnames: + fullpath = os.path.join(root, fname) + self._possibly_add_to_pending_batch(fullpath) + + if self._pending_batch: + self._submit_pending_batch() + + # Tell the world we're done. + with self.condition: + self.done = True + self.condition.notify() + + +def file_batches( + paths: list[str], + batch_size: int = 1, + file_filter: Optional[Callable[[str], bool]] = None, + include_mac_packages: bool = False, +) -> Generator[list[str], None, None]: + """Efficiently yield batches of files to operate on. + + Accepts a list of paths which can be files or directories to be recursed. + The batch lists are buffered in a background thread so time-consuming + synchronous operations on the returned batches will not slow the gather. + """ + + run = _FileBatchesRun(paths=paths, + batch_size=batch_size, + file_filter=file_filter, + include_mac_packages=include_mac_packages) + + # Spin up a bg thread to feed us batches. + thread = Thread(target=run.bg_thread) + thread.start() + + # Now spin waiting for new batches to come in or completion/errors. + while True: + with run.condition: + run.condition.wait_for(lambda: run.done or run.errored or run. + batches) + try: + if run.errored: + raise RuntimeError('BG batch run errored.') + while run.batches: + yield run.batches.pop(0) + if run.done: + break + except GeneratorExit: + # Lets the bg thread know to abort. + run.done = True + raise + finally: + run.condition.notify()