diff --git a/.efrocachemap b/.efrocachemap
index 4395799c..27400acf 100644
--- a/.efrocachemap
+++ b/.efrocachemap
@@ -420,7 +420,7 @@
"assets/build/ba_data/audio/zoeOw.ogg": "https://files.ballistica.net/cache/ba1/60/ad/38269b7f1c7dc20cb9a506cd0681",
"assets/build/ba_data/audio/zoePickup01.ogg": "https://files.ballistica.net/cache/ba1/72/85/d6fc4d16b7081d91fba2850b5b10",
"assets/build/ba_data/audio/zoeScream01.ogg": "https://files.ballistica.net/cache/ba1/e9/ae/1d674d0c086eaa0bd1c3b1db0505",
- "assets/build/ba_data/data/langdata.json": "https://files.ballistica.net/cache/ba1/5a/10/dae249cdd589b795fd341ebbec42",
+ "assets/build/ba_data/data/langdata.json": "https://files.ballistica.net/cache/ba1/7d/d4/6a32da2a6a5d1f8d71f65ac65792",
"assets/build/ba_data/data/languages/arabic.json": "https://files.ballistica.net/cache/ba1/e2/24/5e7ea9ca5c9de4d3b7a28e53564d",
"assets/build/ba_data/data/languages/belarussian.json": "https://files.ballistica.net/cache/ba1/61/03/89070ca765e06da3a419a579f503",
"assets/build/ba_data/data/languages/chinese.json": "https://files.ballistica.net/cache/ba1/f8/15/e1a2fa38697417bcf2cf19cd34ef",
@@ -438,24 +438,24 @@
"assets/build/ba_data/data/languages/greek.json": "https://files.ballistica.net/cache/ba1/e0/04/6be14bff785255719756e0906ea9",
"assets/build/ba_data/data/languages/hindi.json": "https://files.ballistica.net/cache/ba1/91/98/42701cd595c2f70b7484614a8f49",
"assets/build/ba_data/data/languages/hungarian.json": "https://files.ballistica.net/cache/ba1/d8/f2/aa16bc336bd7660cc86c3264bfc4",
- "assets/build/ba_data/data/languages/indonesian.json": "https://files.ballistica.net/cache/ba1/7f/a9/db86d4c8b70f06fd8a1fe0c0511b",
+ "assets/build/ba_data/data/languages/indonesian.json": "https://files.ballistica.net/cache/ba1/82/76/1ca7ba627f34be34961de40fe91f",
"assets/build/ba_data/data/languages/italian.json": "https://files.ballistica.net/cache/ba1/1a/10/9563348e729d1e5c8ae8c9cbc1f2",
- "assets/build/ba_data/data/languages/korean.json": "https://files.ballistica.net/cache/ba1/a8/e9/171a904f1331fdb7b1918a0f2598",
+ "assets/build/ba_data/data/languages/korean.json": "https://files.ballistica.net/cache/ba1/7c/38/d4a44c481757d355836f292ede48",
"assets/build/ba_data/data/languages/persian.json": "https://files.ballistica.net/cache/ba1/df/b1/b2c9ebaad5e873ebedd365726d3d",
"assets/build/ba_data/data/languages/polish.json": "https://files.ballistica.net/cache/ba1/19/e9/59c891b1fb85f3ba9f19283c233d",
- "assets/build/ba_data/data/languages/portuguese.json": "https://files.ballistica.net/cache/ba1/da/95/36797ec53a697a04e55b225a701d",
+ "assets/build/ba_data/data/languages/portuguese.json": "https://files.ballistica.net/cache/ba1/6b/9c/0c8fe0e4d5fc0c29b95ad798ee23",
"assets/build/ba_data/data/languages/romanian.json": "https://files.ballistica.net/cache/ba1/d7/06/9d70642d0a4d1e3b1c2149d7a17c",
"assets/build/ba_data/data/languages/russian.json": "https://files.ballistica.net/cache/ba1/8d/c1/90cc02326100ccee7f03d0cb42b7",
"assets/build/ba_data/data/languages/serbian.json": "https://files.ballistica.net/cache/ba1/4e/91/6f2a9a3ce733908e91377a6ddb9a",
"assets/build/ba_data/data/languages/slovak.json": "https://files.ballistica.net/cache/ba1/f9/4b/d9f01814224066856695452ef57c",
- "assets/build/ba_data/data/languages/spanish.json": "https://files.ballistica.net/cache/ba1/e1/83/07b3561f8b15b782ff968dbaf919",
+ "assets/build/ba_data/data/languages/spanish.json": "https://files.ballistica.net/cache/ba1/12/8e/8bc7f197b725da644ca4869f8854",
"assets/build/ba_data/data/languages/swedish.json": "https://files.ballistica.net/cache/ba1/91/0a/35c4baf539d5951fc03a794c0e0b",
"assets/build/ba_data/data/languages/tamil.json": "https://files.ballistica.net/cache/ba1/94/1a/533bc718e676191bafc25e2dc98f",
"assets/build/ba_data/data/languages/thai.json": "https://files.ballistica.net/cache/ba1/f7/df/7ba5f99c5c2c4c86fc0503fcf0b7",
"assets/build/ba_data/data/languages/turkish.json": "https://files.ballistica.net/cache/ba1/9a/90/8e2ed626def09f88c3b9ab5215a3",
"assets/build/ba_data/data/languages/ukrainian.json": "https://files.ballistica.net/cache/ba1/ab/35/644e4239cfa62a597a905412b90c",
"assets/build/ba_data/data/languages/venetian.json": "https://files.ballistica.net/cache/ba1/53/9e/068074156b38bab7f732977a4031",
- "assets/build/ba_data/data/languages/vietnamese.json": "https://files.ballistica.net/cache/ba1/25/13/b64b849fc9fedcc18d81f6e08c4d",
+ "assets/build/ba_data/data/languages/vietnamese.json": "https://files.ballistica.net/cache/ba1/1f/ae/abe3f105b3c4b51f6b7942773305",
"assets/build/ba_data/data/maps/big_g.json": "https://files.ballistica.net/cache/ba1/47/0a/a617cc85d927b576c4e6fc1091ed",
"assets/build/ba_data/data/maps/bridgit.json": "https://files.ballistica.net/cache/ba1/03/4b/57ee9b42854b26f23f81bd8c58ef",
"assets/build/ba_data/data/maps/courtyard.json": "https://files.ballistica.net/cache/ba1/03/38/344dd05bfef7bbdf464035ec5aa2",
@@ -3995,50 +3995,50 @@
"assets/src/ba_data/python/ba/_generated/__init__.py": "https://files.ballistica.net/cache/ba1/ee/e8/cad05aa531c7faf7ff7b96db7f6e",
"assets/src/ba_data/python/ba/_generated/enums.py": "https://files.ballistica.net/cache/ba1/b2/e5/0ee0561e16257a32830645239f34",
"ballisticacore-windows/Generic/BallisticaCore.ico": "https://files.ballistica.net/cache/ba1/89/c0/e32c7d2a35dc9aef57cc73b0911a",
- "build/prefab/full/linux_arm64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/55/b1/d1c692a3ddbcfec532e71a827f74",
- "build/prefab/full/linux_arm64_gui/release/ballisticacore": "https://files.ballistica.net/cache/ba1/e4/02/9697c22bdc862cf4024da4291a67",
- "build/prefab/full/linux_arm64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/92/52/03c71172f9ef4ebaf62e7b61fa3b",
- "build/prefab/full/linux_arm64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/1e/ea/529dc5f93a4597eb32dbc6adc5fc",
- "build/prefab/full/linux_x86_64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/65/0c/75a9348dab828ae8c35f4ea4a5af",
- "build/prefab/full/linux_x86_64_gui/release/ballisticacore": "https://files.ballistica.net/cache/ba1/8b/6c/21066172ee06ba37a9a65ad8201f",
- "build/prefab/full/linux_x86_64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/96/32/bc967f00d76dbef3df1a8580cfdd",
- "build/prefab/full/linux_x86_64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/63/73/09ea54fc26fa042970c298ced100",
- "build/prefab/full/mac_arm64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/d6/11/859b3a49d2d76f83318708146f78",
- "build/prefab/full/mac_arm64_gui/release/ballisticacore": "https://files.ballistica.net/cache/ba1/ec/92/b722ad7712ee2ecc15a595c0b152",
- "build/prefab/full/mac_arm64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/ee/43/67c17b0f8f5c6aad23c6eec0a4dc",
- "build/prefab/full/mac_arm64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/29/c0/988af43d3cf5a22a8959d0c26aa0",
- "build/prefab/full/mac_x86_64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/6e/ea/7ae615507621320412e5e3d71052",
- "build/prefab/full/mac_x86_64_gui/release/ballisticacore": "https://files.ballistica.net/cache/ba1/90/40/4e57c929d175668f19ff52b14a80",
- "build/prefab/full/mac_x86_64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/67/43/7bb14e1ddb12ac6f321d02def119",
- "build/prefab/full/mac_x86_64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/6e/f7/b892e1e69f7605e6ce3099c80ff6",
- "build/prefab/full/windows_x86_gui/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/d6/fe/037d6240e9b41f82e6d5b51b9e3a",
- "build/prefab/full/windows_x86_gui/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/cc/c7/63b1d1fa549417fb4f9a50704e09",
- "build/prefab/full/windows_x86_server/debug/dist/BallisticaCoreHeadless.exe": "https://files.ballistica.net/cache/ba1/06/f9/3f412708758f422557b56e3139a6",
- "build/prefab/full/windows_x86_server/release/dist/BallisticaCoreHeadless.exe": "https://files.ballistica.net/cache/ba1/9f/c3/cc12905869eba2278e4021316233",
- "build/prefab/lib/linux_arm64_gui/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/70/9d/82e666398551b30fc854ed62682c",
- "build/prefab/lib/linux_arm64_gui/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/6f/82/ae4fb4b892054b1e09f2d0dd430b",
- "build/prefab/lib/linux_arm64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/5b/c3/71ef621365cb3cd5d05c770ba342",
- "build/prefab/lib/linux_arm64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/ed/c3/332d53ad4beb62b4b315d5f14b5c",
- "build/prefab/lib/linux_x86_64_gui/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/35/f0/84557f7b3159a7f4600ee619689b",
- "build/prefab/lib/linux_x86_64_gui/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/ed/64/1ef3f2dab606a16586c5ace3b57f",
- "build/prefab/lib/linux_x86_64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/9a/4d/5249822bd9c9e9b39ae8f635bfbb",
- "build/prefab/lib/linux_x86_64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/cf/81/0187c04509a2e6eb1bbf82cf634d",
- "build/prefab/lib/mac_arm64_gui/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/dc/ac/782541d23b9419b1eeda8d17bd58",
- "build/prefab/lib/mac_arm64_gui/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/a7/ad/09b4bbe131cb94bf7e63dba25d84",
- "build/prefab/lib/mac_arm64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/74/c4/191772694cf5b8266228f2608f79",
- "build/prefab/lib/mac_arm64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/d2/81/7ad01e29e031f2fa2f48ae4663ba",
- "build/prefab/lib/mac_x86_64_gui/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/41/9e/e65888340a6a98d2c2e51bce0ea9",
- "build/prefab/lib/mac_x86_64_gui/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/b3/a4/8781964774ecfcb2d5c84029d9c9",
- "build/prefab/lib/mac_x86_64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/08/a3/709ddff4335eff11913c75892ac2",
- "build/prefab/lib/mac_x86_64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/5f/46/da058d1d0c43e7193275f3970d46",
- "build/prefab/lib/windows/Debug_Win32/BallisticaCoreGenericInternal.lib": "https://files.ballistica.net/cache/ba1/87/60/f4b7c14aeac9e6e14ecb564608c6",
- "build/prefab/lib/windows/Debug_Win32/BallisticaCoreGenericInternal.pdb": "https://files.ballistica.net/cache/ba1/72/b1/0fc35401b8475d86b20cc138ab40",
- "build/prefab/lib/windows/Debug_Win32/BallisticaCoreHeadlessInternal.lib": "https://files.ballistica.net/cache/ba1/aa/ce/22120a61f7b9dce0d822dc4e8794",
- "build/prefab/lib/windows/Debug_Win32/BallisticaCoreHeadlessInternal.pdb": "https://files.ballistica.net/cache/ba1/e5/1c/b52a0879d61e86a2d550b3882682",
- "build/prefab/lib/windows/Release_Win32/BallisticaCoreGenericInternal.lib": "https://files.ballistica.net/cache/ba1/c1/36/db7c22ed5a386c6ea29c59ffc20e",
- "build/prefab/lib/windows/Release_Win32/BallisticaCoreGenericInternal.pdb": "https://files.ballistica.net/cache/ba1/77/26/93d4d7345649dd59beb21df0a0e6",
- "build/prefab/lib/windows/Release_Win32/BallisticaCoreHeadlessInternal.lib": "https://files.ballistica.net/cache/ba1/e2/bd/7cc56c36d6d45f6c29c37a3d2874",
- "build/prefab/lib/windows/Release_Win32/BallisticaCoreHeadlessInternal.pdb": "https://files.ballistica.net/cache/ba1/b7/47/662d87a97f3dc85e34b046b50e98",
+ "build/prefab/full/linux_arm64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/b4/b2/d9d81b227c329f77198d96ee2ae1",
+ "build/prefab/full/linux_arm64_gui/release/ballisticacore": "https://files.ballistica.net/cache/ba1/06/0d/c5503c7e9b4e8c5465b7df36c3ab",
+ "build/prefab/full/linux_arm64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/2e/57/6c8e84496af8bcfc60b6030f9008",
+ "build/prefab/full/linux_arm64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/f1/37/e421b0f64743dd33ff9e3db54838",
+ "build/prefab/full/linux_x86_64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/3a/83/153f01e88ee01db4e2dd234ce189",
+ "build/prefab/full/linux_x86_64_gui/release/ballisticacore": "https://files.ballistica.net/cache/ba1/0f/9b/66b3b2c089137e81496868f9c829",
+ "build/prefab/full/linux_x86_64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/5a/2e/0b387964a1cbe7658f353b0c3d58",
+ "build/prefab/full/linux_x86_64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/77/52/ae35cb8cfea02c60417968702b3f",
+ "build/prefab/full/mac_arm64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/cf/55/d502cfa9bef1142b7cc240759c07",
+ "build/prefab/full/mac_arm64_gui/release/ballisticacore": "https://files.ballistica.net/cache/ba1/c7/59/4d6c76a8e57478a81804b446cec7",
+ "build/prefab/full/mac_arm64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/ed/5c/68871f87dd6353ca6693b24c895c",
+ "build/prefab/full/mac_arm64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/69/34/d573542acf00286337a8a7d4070e",
+ "build/prefab/full/mac_x86_64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/54/21/ad891fc900eb06eb307401887479",
+ "build/prefab/full/mac_x86_64_gui/release/ballisticacore": "https://files.ballistica.net/cache/ba1/10/6d/b079707d97db261651539cf42be4",
+ "build/prefab/full/mac_x86_64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/9f/ec/d6f090057a354d6f27778a79151d",
+ "build/prefab/full/mac_x86_64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/2f/8d/7076055efd990269b20af15d092a",
+ "build/prefab/full/windows_x86_gui/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/b6/9b/04419fdb2911f107ba2cf3d1daef",
+ "build/prefab/full/windows_x86_gui/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/89/c0/d3b177978d11d0283b0290e88960",
+ "build/prefab/full/windows_x86_server/debug/dist/BallisticaCoreHeadless.exe": "https://files.ballistica.net/cache/ba1/32/23/ce5919233ff0438442f6e6e66a62",
+ "build/prefab/full/windows_x86_server/release/dist/BallisticaCoreHeadless.exe": "https://files.ballistica.net/cache/ba1/99/51/13dbd2e177dd17314b4bc86fe200",
+ "build/prefab/lib/linux_arm64_gui/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/e2/88/53757bc9fd92d49bd35dc6d3be0e",
+ "build/prefab/lib/linux_arm64_gui/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/06/69/078f7eef49a1127a1492db4703f6",
+ "build/prefab/lib/linux_arm64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/82/f1/2b13fe77164f72d2bf57453bb8e5",
+ "build/prefab/lib/linux_arm64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/6a/b0/a853b61ab794706bbf395ecd2a80",
+ "build/prefab/lib/linux_x86_64_gui/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/76/3d/b0d2913a1650bdc35b2ca0d81154",
+ "build/prefab/lib/linux_x86_64_gui/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/cb/b3/39a0642a376e1f131172f9500353",
+ "build/prefab/lib/linux_x86_64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/7a/07/4804a222e0d92f0fab8b279ce4c1",
+ "build/prefab/lib/linux_x86_64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/55/09/bf8e7d6ce41962163411c6bbd884",
+ "build/prefab/lib/mac_arm64_gui/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/1d/be/5e0b2be7272c4e443cc974d5b182",
+ "build/prefab/lib/mac_arm64_gui/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/5e/ab/075e9137d21e6110d29b67210533",
+ "build/prefab/lib/mac_arm64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/54/2a/c5e1d5ed4328c40821695db2cd84",
+ "build/prefab/lib/mac_arm64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/b7/dd/55210b3f3c075d9b237e0c6aa733",
+ "build/prefab/lib/mac_x86_64_gui/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/a2/00/d01842e8b0777f7e6ea47c912b16",
+ "build/prefab/lib/mac_x86_64_gui/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/a6/91/f9cb15d0876750e28abe3b0d221c",
+ "build/prefab/lib/mac_x86_64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/b7/11/1ecfe322ae997772b71538664cad",
+ "build/prefab/lib/mac_x86_64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/8d/d6/1e83dba73d581cfb2b2f6eb31f22",
+ "build/prefab/lib/windows/Debug_Win32/BallisticaCoreGenericInternal.lib": "https://files.ballistica.net/cache/ba1/50/6a/ca5e49b3cad047b541648dc9914d",
+ "build/prefab/lib/windows/Debug_Win32/BallisticaCoreGenericInternal.pdb": "https://files.ballistica.net/cache/ba1/6f/f7/b38282cfbd3cb0cda89d5a458176",
+ "build/prefab/lib/windows/Debug_Win32/BallisticaCoreHeadlessInternal.lib": "https://files.ballistica.net/cache/ba1/97/e5/1d2e76fcadbe022d4c522c7d2135",
+ "build/prefab/lib/windows/Debug_Win32/BallisticaCoreHeadlessInternal.pdb": "https://files.ballistica.net/cache/ba1/54/d6/2d642477837c34c946b70e27014d",
+ "build/prefab/lib/windows/Release_Win32/BallisticaCoreGenericInternal.lib": "https://files.ballistica.net/cache/ba1/79/2b/f17b81bdf03719098a558305f0a3",
+ "build/prefab/lib/windows/Release_Win32/BallisticaCoreGenericInternal.pdb": "https://files.ballistica.net/cache/ba1/9a/5a/3ca5679187f8dd5f89d8bb68ed84",
+ "build/prefab/lib/windows/Release_Win32/BallisticaCoreHeadlessInternal.lib": "https://files.ballistica.net/cache/ba1/4b/88/d016d4059ea7b334e95d1e6ef258",
+ "build/prefab/lib/windows/Release_Win32/BallisticaCoreHeadlessInternal.pdb": "https://files.ballistica.net/cache/ba1/1b/35/b11d851fc912b7bcd57766981fa2",
"src/ballistica/generated/python_embedded/binding.inc": "https://files.ballistica.net/cache/ba1/c0/32/b7907e3859a5c5013a3d97b6b523",
"src/ballistica/generated/python_embedded/bootstrap.inc": "https://files.ballistica.net/cache/ba1/2d/4f/f4fe67827f36cd59cd5193333a02",
"src/ballistica/generated/python_embedded/bootstrap_monolithic.inc": "https://files.ballistica.net/cache/ba1/ef/c1/aa5f1aa10af89f5c0b1e616355fd"
diff --git a/.idea/dictionaries/ericf.xml b/.idea/dictionaries/ericf.xml
index 7861d0c9..17bdf891 100644
--- a/.idea/dictionaries/ericf.xml
+++ b/.idea/dictionaries/ericf.xml
@@ -65,6 +65,8 @@
alarmsound
alibaba
allerrors
+ allobjc
+ allobjs
allpaths
allsettings
allteams
@@ -603,6 +605,7 @@
depsval
dereferencing
descpos
+ desctype
dest
destdir
devel
@@ -1046,9 +1049,12 @@
getname
getnodes
getnodetype
+ getobj
+ getobjs
getopt
getplayer
getpt
+ getrefs
getremote
getres
getscanresults
@@ -1719,11 +1725,13 @@
nvidia
nyko
obj's
+ objid
objname
objs
objt
objtoyaml
objtype
+ objtypes
obval
occurrances
oculus
@@ -1731,6 +1739,7 @@
offsanchor
offsx
offsy
+ ofile
ofval
oggenc
oghash
@@ -1794,6 +1803,8 @@
packagepathstr
packageversion
painttxtattr
+ pairsj
+ pairss
palmos
pandoc
pandroid
@@ -1944,9 +1955,12 @@
priceraw
printcolors
printf
+ printfiles
printnodes
printobjects
printpaths
+ printrefs
+ printtypes
priv
privatetab
proactor
@@ -2627,9 +2641,14 @@
tpimport
tpimportex
tpimports
+ tpitem
tplayer
+ tpname
tpos
tproxy
+ tpsj
+ tpss
+ tpval
tracebacks
tracemalloc
tradeoff
@@ -2644,6 +2663,7 @@
trynum
tscale
tscl
+ tsed
tself
tspc
tsrcpath
@@ -2843,6 +2863,7 @@
wsroot
wtcolor
wtflib
+ wtfslice
wttxt
wvmpth
xach
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 36d71f1a..9b9923e8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,8 +1,9 @@
-### 1.7.10 (build 20882, api 7, 2022-09-24)
+### 1.7.10 (build 20887, api 7, 2022-09-29)
- Added eval support for cloud-console. This means you can type something like '1+1' in the console and see '2' printed. This is how Python behaves in the stdin console or in-game console or the standard Python interpreter.
- Exceptions in the cloud-console now print to stderr instead of logging.exception(). This means they aren't a pretty red color anymore, but this will keep cloud-console behaving well with things like servers where logging.exception() might trigger alarms or otherwise. This is also consistent with standard interactive Python behavior.
- Cloud console now shows the device name at the top instead of simply 'Console' while connected.
- Moved the function that actually runs cloud console code to `ba._cloud.cloud_console_exec()`.
+- Added efro.debug which contains useful functionality for debugging object reference issues and memory leaks on live app instances (via cloud shell or whatever).
### 1.7.9 (build 20880, api 7, 2022-09-24)
- Cleaned up the efro.message system to isolate response types that are used purely internally (via a new SysResponse type).
diff --git a/assets/.asset_manifest_public.json b/assets/.asset_manifest_public.json
index daf534f9..460603ae 100644
--- a/assets/.asset_manifest_public.json
+++ b/assets/.asset_manifest_public.json
@@ -65,7 +65,6 @@
"ba_data/python/ba/__pycache__/_tournament.cpython-310.opt-1.pyc",
"ba_data/python/ba/__pycache__/_ui.cpython-310.opt-1.pyc",
"ba_data/python/ba/__pycache__/_workspace.cpython-310.opt-1.pyc",
- "ba_data/python/ba/__pycache__/deprecated.cpython-310.opt-1.pyc",
"ba_data/python/ba/__pycache__/internal.cpython-310.opt-1.pyc",
"ba_data/python/ba/__pycache__/macmusicapp.cpython-310.opt-1.pyc",
"ba_data/python/ba/__pycache__/modutils.cpython-310.opt-1.pyc",
@@ -136,7 +135,6 @@
"ba_data/python/ba/_tournament.py",
"ba_data/python/ba/_ui.py",
"ba_data/python/ba/_workspace.py",
- "ba_data/python/ba/deprecated.py",
"ba_data/python/ba/internal.py",
"ba_data/python/ba/macmusicapp.py",
"ba_data/python/ba/modutils.py",
@@ -516,6 +514,7 @@
"ba_data/python/efro/__init__.py",
"ba_data/python/efro/__pycache__/__init__.cpython-310.opt-1.pyc",
"ba_data/python/efro/__pycache__/call.cpython-310.opt-1.pyc",
+ "ba_data/python/efro/__pycache__/debug.cpython-310.opt-1.pyc",
"ba_data/python/efro/__pycache__/error.cpython-310.opt-1.pyc",
"ba_data/python/efro/__pycache__/log.cpython-310.opt-1.pyc",
"ba_data/python/efro/__pycache__/rpc.cpython-310.opt-1.pyc",
@@ -538,6 +537,7 @@
"ba_data/python/efro/dataclassio/_pathcapture.py",
"ba_data/python/efro/dataclassio/_prep.py",
"ba_data/python/efro/dataclassio/extras.py",
+ "ba_data/python/efro/debug.py",
"ba_data/python/efro/error.py",
"ba_data/python/efro/log.py",
"ba_data/python/efro/message/__init__.py",
diff --git a/assets/Makefile b/assets/Makefile
index d9c89f72..07334f66 100644
--- a/assets/Makefile
+++ b/assets/Makefile
@@ -199,7 +199,6 @@ SCRIPT_TARGETS_PY_PUBLIC = \
build/ba_data/python/ba/_tournament.py \
build/ba_data/python/ba/_ui.py \
build/ba_data/python/ba/_workspace.py \
- build/ba_data/python/ba/deprecated.py \
build/ba_data/python/ba/internal.py \
build/ba_data/python/ba/macmusicapp.py \
build/ba_data/python/ba/modutils.py \
@@ -451,7 +450,6 @@ SCRIPT_TARGETS_PYC_PUBLIC = \
build/ba_data/python/ba/__pycache__/_tournament.cpython-310.opt-1.pyc \
build/ba_data/python/ba/__pycache__/_ui.cpython-310.opt-1.pyc \
build/ba_data/python/ba/__pycache__/_workspace.cpython-310.opt-1.pyc \
- build/ba_data/python/ba/__pycache__/deprecated.cpython-310.opt-1.pyc \
build/ba_data/python/ba/__pycache__/internal.cpython-310.opt-1.pyc \
build/ba_data/python/ba/__pycache__/macmusicapp.cpython-310.opt-1.pyc \
build/ba_data/python/ba/__pycache__/modutils.cpython-310.opt-1.pyc \
@@ -669,6 +667,7 @@ SCRIPT_TARGETS_PY_PUBLIC_TOOLS = \
build/ba_data/python/efro/dataclassio/_pathcapture.py \
build/ba_data/python/efro/dataclassio/_prep.py \
build/ba_data/python/efro/dataclassio/extras.py \
+ build/ba_data/python/efro/debug.py \
build/ba_data/python/efro/error.py \
build/ba_data/python/efro/log.py \
build/ba_data/python/efro/message/__init__.py \
@@ -700,6 +699,7 @@ SCRIPT_TARGETS_PYC_PUBLIC_TOOLS = \
build/ba_data/python/efro/dataclassio/__pycache__/_pathcapture.cpython-310.opt-1.pyc \
build/ba_data/python/efro/dataclassio/__pycache__/_prep.cpython-310.opt-1.pyc \
build/ba_data/python/efro/dataclassio/__pycache__/extras.cpython-310.opt-1.pyc \
+ build/ba_data/python/efro/__pycache__/debug.cpython-310.opt-1.pyc \
build/ba_data/python/efro/__pycache__/error.cpython-310.opt-1.pyc \
build/ba_data/python/efro/__pycache__/log.cpython-310.opt-1.pyc \
build/ba_data/python/efro/message/__pycache__/__init__.cpython-310.opt-1.pyc \
diff --git a/assets/src/ba_data/python/._ba_sources_hash b/assets/src/ba_data/python/._ba_sources_hash
index ac896033..861b23df 100644
--- a/assets/src/ba_data/python/._ba_sources_hash
+++ b/assets/src/ba_data/python/._ba_sources_hash
@@ -1 +1 @@
-69724857583156237926512795146611373217
\ No newline at end of file
+194057364831757023796080999188881665880
\ No newline at end of file
diff --git a/assets/src/ba_data/python/ba/_bootstrap.py b/assets/src/ba_data/python/ba/_bootstrap.py
index 88a4a4eb..6fe2b550 100644
--- a/assets/src/ba_data/python/ba/_bootstrap.py
+++ b/assets/src/ba_data/python/ba/_bootstrap.py
@@ -45,7 +45,7 @@ def bootstrap() -> None:
# Give a soft warning if we're being used with a different binary
# version than we expect.
- expected_build = 20882
+ expected_build = 20887
running_build: int = env['build_number']
if running_build != expected_build:
print(
diff --git a/assets/src/ba_data/python/ba/deprecated.py b/assets/src/ba_data/python/ba/deprecated.py
deleted file mode 100644
index de3ff2aa..00000000
--- a/assets/src/ba_data/python/ba/deprecated.py
+++ /dev/null
@@ -1,8 +0,0 @@
-# Released under the MIT License. See LICENSE for details.
-#
-"""Deprecated functionality.
-
-Classes or functions can be relocated here when they are deprecated.
-Any code using them should migrate to alternative methods, as
-deprecated items will eventually be fully removed.
-"""
diff --git a/ballisticacore-cmake/.idea/dictionaries/ericf.xml b/ballisticacore-cmake/.idea/dictionaries/ericf.xml
index 361b5efa..ad6d8d05 100644
--- a/ballisticacore-cmake/.idea/dictionaries/ericf.xml
+++ b/ballisticacore-cmake/.idea/dictionaries/ericf.xml
@@ -42,6 +42,8 @@
alext
alibaba
allerrors
+ allobjc
+ allobjs
allocs
allwarnings
alot
@@ -328,6 +330,7 @@
demangling
denom
dernit
+ desctype
destdir
dets
dfba
@@ -545,6 +548,8 @@
getname
getnodes
getnodetype
+ getobj
+ getobjs
getpackagecollidemodel
getpackagedata
getpackagemodel
@@ -553,6 +558,7 @@
getpublicpartyenabled
getpublicpartymaxsize
getqrcodetexture
+ getrefs
getres
getsession
getsound
@@ -903,12 +909,14 @@
objexists
objid
objtoyaml
+ objtypes
obstack
obvs
oculus
oenval
offsx
offsy
+ ofile
oiffsss
okbtn
oldbook
@@ -955,6 +963,8 @@
outvalue
ouya
ovld
+ pairsj
+ pairss
parameteriv
passcode
pathcapture
@@ -1009,8 +1019,11 @@
prerun
prettypath
printf
+ printfiles
printnodes
printobjects
+ printrefs
+ printtypes
priv
privatetab
processinfoplistfile
@@ -1347,6 +1360,11 @@
tpimport
tpimportex
tpimports
+ tpitem
+ tpname
+ tpsj
+ tpss
+ tpval
tracebacks
tracestr
trackpad
@@ -1361,6 +1379,7 @@
trimesh
trimeshes
trynum
+ tsed
tself
tsrcpath
tunmd
@@ -1469,6 +1488,7 @@
writeauxiliaryfile
wspath
wsroot
+ wtfslice
wunused
wvmpth
xcframework
diff --git a/config/config.json b/config/config.json
index 5b6c3394..1fcd9866 100644
--- a/config/config.json
+++ b/config/config.json
@@ -35,7 +35,8 @@
"filelock",
"Cocoa",
"pdoc",
- "certifi"
+ "certifi",
+ "psutil"
],
"python_paths": [
"assets/src/ba_data/python",
diff --git a/config/toolconfigsrc/mypy.ini b/config/toolconfigsrc/mypy.ini
index 066aed9e..5e5d14a4 100644
--- a/config/toolconfigsrc/mypy.ini
+++ b/config/toolconfigsrc/mypy.ini
@@ -16,6 +16,9 @@ no_implicit_reexport = False
[mypy-ba.deprecated]
no_implicit_reexport = False
+[mypy-psutil]
+ignore_missing_imports = True
+
[mypy-Cocoa.*]
ignore_missing_imports = True
@@ -39,3 +42,4 @@ disallow_any_unimported = False
[mypy-pdoc]
ignore_missing_imports = True
+
diff --git a/src/ballistica/ballistica.cc b/src/ballistica/ballistica.cc
index 22d83546..99446c21 100644
--- a/src/ballistica/ballistica.cc
+++ b/src/ballistica/ballistica.cc
@@ -32,7 +32,7 @@
namespace ballistica {
// These are set automatically via script; don't modify them here.
-const int kAppBuildNumber = 20882;
+const int kAppBuildNumber = 20887;
const char* kAppVersion = "1.7.10";
// Our standalone globals.
diff --git a/src/ballistica/python/python_context_call.cc b/src/ballistica/python/python_context_call.cc
index 938e2dd1..3a3980cc 100644
--- a/src/ballistica/python/python_context_call.cc
+++ b/src/ballistica/python/python_context_call.cc
@@ -10,7 +10,7 @@
namespace ballistica {
// FIXME - should be static member var
-PythonContextCall* PythonContextCall::current_call_ = nullptr;
+PythonContextCall* PythonContextCall::current_call_{};
PythonContextCall::PythonContextCall(PyObject* obj_in) {
assert(InLogicThread());
diff --git a/tools/efro/debug.py b/tools/efro/debug.py
new file mode 100644
index 00000000..0b9b12a3
--- /dev/null
+++ b/tools/efro/debug.py
@@ -0,0 +1,241 @@
+# Released under the MIT License. See LICENSE for details.
+#
+"""Utilities for debugging memory leaks or other issues."""
+from __future__ import annotations
+
+import gc
+import sys
+import types
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from typing import Any, TextIO
+
+ABS_MAX_LEVEL = 10
+
+# NOTE: In general we want this toolset to allow us to explore
+# which objects are holding references to others so we can diagnose
+# leaks/etc. It is a bit tricky to do that, however, without
+# affecting the objects we are looking at by adding temporary references
+# from module dicts, function scopes, etc. So we need to try to be
+# careful about cleaning up after ourselves and explicitly avoiding
+# returning these temporary references wherever possible.
+
+# A good test is running printrefs() repeatedly on some object that is
+# known to be static. If the list of references or the ids or any
+# the listed references changes with each run, it's a good sign that
+# we're showing some temporary objects that we should be ignoring.
+
+
+def getobjs(cls: type | str, contains: str | None = None) -> list[Any]:
+ """Return all garbage-collected objects matching criteria.
+
+ 'type' can be an actual type or a string in which case objects
+ whose types contain that string will be returned.
+
+ If 'contains' is provided, objects will be filtered to those
+ containing that in their str() representations.
+ """
+
+ # Don't wanna return stuff waiting to be garbage-collected.
+ gc.collect()
+
+ if not isinstance(cls, type | str):
+ raise TypeError('Expected a type or string for cls')
+ if not isinstance(contains, str | None):
+ raise TypeError('Expected a string or None for contains')
+
+ if isinstance(cls, str):
+ objs = [o for o in gc.get_objects() if cls in str(type(o))]
+ else:
+ objs = [o for o in gc.get_objects() if isinstance(o, cls)]
+ if contains is not None:
+ objs = [o for o in objs if contains in str(o)]
+
+ return objs
+
+
+def getobj(objid: int) -> Any:
+ """Return a garbage-collected object by its id.
+
+ Remember that this is VERY inefficient and should only ever be used
+ for debugging.
+ """
+ if not isinstance(objid, int):
+ raise TypeError(f'Expected an int for objid; got a {type(objid)}.')
+
+ # Don't wanna return stuff waiting to be garbage-collected.
+ for obj in gc.get_objects():
+ if id(obj) == objid:
+ return obj
+ raise RuntimeError(f'Object with id {objid} not found.')
+
+
+def getrefs(obj: Any) -> list[Any]:
+ """Given an object, return things referencing it."""
+ v = vars() # Ignore ref coming from locals.
+ return [o for o in gc.get_referrers(obj) if o is not v]
+
+
+def printfiles(file: TextIO | None = None) -> None:
+ """Print info about open files in the current app."""
+ import io
+ file = sys.stderr if file is None else file
+ try:
+ import psutil
+ except ImportError:
+ print(
+ "Error: printfiles requires the 'psutil' module to be installed.",
+ file=file)
+ return
+
+ proc = psutil.Process()
+
+ # Let's grab all Python file handles so we can associate raw files
+ # with their Python objects when possible.
+ fileio_ids = {obj.fileno(): obj for obj in getobjs(io.FileIO)}
+ textio_ids = {obj.fileno(): obj for obj in getobjs(io.TextIOWrapper)}
+
+ # FIXME: we could do a more limited version of this when psutil is
+ # not present that simply includes Python's files.
+ print('Files open by this app (not limited to Python\'s):', file=file)
+ for i, ofile in enumerate(proc.open_files()):
+ # Mypy doesn't know about mode apparently.
+ # (and can't use type: ignore because we don't require psutil
+ # and then mypy complains about unused ignore comment when its
+ # not present)
+ mode = getattr(ofile, 'mode')
+ assert isinstance(mode, str)
+ textio = textio_ids.get(ofile.fd)
+ textio_s = id(textio) if textio is not None else ''
+ fileio = fileio_ids.get(ofile.fd)
+ fileio_s = id(fileio) if fileio is not None else ''
+ print(f'#{i+1}: path={ofile.path!r},'
+ f' fd={ofile.fd}, mode={mode!r}, TextIOWrapper={textio_s},'
+ f' FileIO={fileio_s}')
+
+
+def printrefs(obj: Any,
+ max_level: int = 2,
+ exclude_objs: list[Any] | None = None,
+ expand_ids: list[int] | None = None,
+ file: TextIO | None = None) -> None:
+ """Print human readable list of objects referring to an object.
+
+ 'max_level' specifies how many levels of recursion are printed.
+ 'exclude_objs' can be a list of exact objects to skip if found in the
+ referrers list. This can be useful to avoid printing the local context
+ where the object was passed in from (locals(), etc).
+ 'expand_ids' can be a list of object ids; if that particular object is
+ found, it will always be expanded even if max_level has been reached.
+ """
+ _printrefs(obj,
+ level=0,
+ max_level=max_level,
+ exclude_objs=[] if exclude_objs is None else exclude_objs,
+ expand_ids=[] if expand_ids is None else expand_ids,
+ file=sys.stderr if file is None else file)
+
+
+def printtypes(limit: int = 50, file: TextIO | None = None) -> None:
+ """Print a human readable list of which types have the most instances."""
+ assert limit > 0
+ objtypes: dict[str, int] = {}
+ gc.collect() # Recommended before get_objects().
+ allobjs = gc.get_objects()
+ allobjc = len(allobjs)
+ for obj in allobjs:
+ modname = type(obj).__module__
+ tpname = type(obj).__qualname__
+ if modname != 'builtins':
+ tpname = f'{modname}.{tpname}'
+ objtypes[tpname] = objtypes.get(tpname, 0) + 1
+
+ # Presumably allobjs contains stack-frame/dict type stuff
+ # from this function call which in turn contain refs to allobjs.
+ # Let's try to prevent these huge lists from accumulating until
+ # the cyclical collector (hopefully) gets to them.
+ allobjs.clear()
+ del allobjs
+
+ print(f'Types most allocated ({allobjc} total objects):', file=file)
+ for i, tpitem in enumerate(
+ sorted(objtypes.items(), key=lambda x: x[1],
+ reverse=True)[:limit]):
+ tpname, tpval = tpitem
+ percent = tpval / allobjc * 100.0
+ print(f'{i+1}: {tpname}: {tpval} ({percent:.2f}%)', file=file)
+
+
+def _desctype(obj: Any) -> str:
+ cls = type(obj)
+ if cls is types.ModuleType:
+ return f'{type(obj).__name__} {obj.__name__}'
+ if cls is types.MethodType:
+ bnd = 'bound' if hasattr(obj, '__self__') else 'unbound'
+ return f'{bnd} {type(obj).__name__} {obj.__name__}'
+ return f'{type(obj).__name__}'
+
+
+def _desc(obj: Any) -> str:
+ extra: str | None = None
+ if isinstance(obj, list | tuple):
+ # Print length and the first few types.
+ tps = [_desctype(i) for i in obj[:3]]
+ tpsj = ', '.join(tps)
+ tpss = (f', contains [{tpsj}, ...]'
+ if len(obj) > 3 else f', contains [{tpsj}]' if tps else '')
+ extra = f' (len {len(obj)}{tpss})'
+ elif isinstance(obj, dict):
+ # If it seems to be the vars() for a type or module,
+ # try to identify what.
+ for ref in getrefs(obj):
+ if hasattr(ref, '__dict__') and vars(ref) is obj:
+ extra = f' (vars for {_desctype(ref)} @ {id(ref)})'
+
+ # Generic dict: print length and the first few key:type pairs.
+ if extra is None:
+ pairs = [
+ f'{repr(n)}: {_desctype(v)}' for n, v in list(obj.items())[:3]
+ ]
+ pairsj = ', '.join(pairs)
+ pairss = (f', contains {{{pairsj}, ...}}' if len(obj) > 3 else
+ f', contains {{{pairsj}}}' if pairs else '')
+ extra = f' (len {len(obj)}{pairss})'
+ if extra is None:
+ extra = ''
+ return f'{_desctype(obj)} @ {id(obj)}{extra}'
+
+
+def _printrefs(obj: Any, level: int, max_level: int, exclude_objs: list,
+ expand_ids: list[int], file: TextIO) -> None:
+ ind = ' ' * level
+ print(ind + _desc(obj), file=file)
+ v = vars()
+ if level < max_level or (id(obj) in expand_ids and level < ABS_MAX_LEVEL):
+ refs = getrefs(obj)
+ for ref in refs:
+
+ # It seems we tend to get a transient cell object with contents
+ # set to obj. Would be nice to understand why that happens
+ # but just ignoring it for now.
+ if isinstance(ref, types.CellType) and ref.cell_contents is obj:
+ continue
+
+ # Ignore anything we were asked to ignore.
+ if exclude_objs is not None:
+ if any(ref is eobj for eobj in exclude_objs):
+ continue
+
+ # Ignore references from our locals.
+ if ref is v:
+ continue
+
+ # The 'refs' list we just made will be listed as a referrer
+ # of this obj, so explicitly exclude it from the obj's listing.
+ _printrefs(ref,
+ level=level + 1,
+ max_level=max_level,
+ exclude_objs=exclude_objs + [refs],
+ expand_ids=expand_ids,
+ file=file)
diff --git a/tools/efro/error.py b/tools/efro/error.py
index d33c42db..e1045580 100644
--- a/tools/efro/error.py
+++ b/tools/efro/error.py
@@ -62,9 +62,17 @@ class RemoteError(Exception):
as a catch-all.
"""
+ def __init__(self, msg: str, peer_desc: str):
+ super().__init__(msg)
+ self._peer_desc = peer_desc
+
def __str__(self) -> str:
s = ''.join(str(arg) for arg in self.args)
- return f'Remote Exception Follows:\n{s}'
+ # Indent so we can more easily tell what is the remote part when
+ # this is in the middle of a long exception chain.
+ padding = ' '
+ s = ''.join(padding + line for line in s.splitlines(keepends=True))
+ return f'The following occurred on {self._peer_desc}:\n{s}'
class IntegrityError(ValueError):
diff --git a/tools/efro/message/_sender.py b/tools/efro/message/_sender.py
index 48a32e9f..47e1dc58 100644
--- a/tools/efro/message/_sender.py
+++ b/tools/efro/message/_sender.py
@@ -48,6 +48,7 @@ class MessageSender:
None] | None = None
self._decode_filter_call: Callable[
[Any, Message, dict, Response | SysResponse], None] | None = None
+ self._peer_desc_call: Callable[[Any], str] | None = None
def send_method(
self, call: Callable[[Any, str],
@@ -102,9 +103,20 @@ class MessageSender:
self._decode_filter_call = call
return call
+ def peer_desc_method(self, call: Callable[[Any],
+ str]) -> Callable[[Any], str]:
+ """Function decorator for defining peer descriptions.
+
+ These are included in error messages or other diagnostics.
+ """
+ assert self._peer_desc_call is None
+ self._peer_desc_call = call
+ return call
+
def send(self, bound_obj: Any, message: Message) -> Response | None:
"""Send a message synchronously."""
return self.send_split_part_2(
+ bound_obj=bound_obj,
message=message,
raw_response=self.send_split_part_1(
bound_obj=bound_obj,
@@ -116,6 +128,7 @@ class MessageSender:
message: Message) -> Response | None:
"""Send a message asynchronously."""
return self.send_split_part_2(
+ bound_obj=bound_obj,
message=message,
raw_response=await self.send_split_part_1_async(
bound_obj=bound_obj,
@@ -178,7 +191,7 @@ class MessageSender:
return self._decode_raw_response(bound_obj, message, response_encoded)
def send_split_part_2(
- self, message: Message,
+ self, bound_obj: Any, message: Message,
raw_response: Response | SysResponse) -> Response | None:
"""Complete message sending (both sync and async).
@@ -186,7 +199,7 @@ class MessageSender:
for when message sending and response handling need to happen
in different contexts/threads.
"""
- response = self._unpack_raw_response(raw_response)
+ response = self._unpack_raw_response(bound_obj, raw_response)
assert (response is None
or type(response) in type(message).get_response_types())
return response
@@ -228,7 +241,8 @@ class MessageSender:
return response
def _unpack_raw_response(
- self, raw_response: Response | SysResponse) -> Response | None:
+ self, bound_obj: Any,
+ raw_response: Response | SysResponse) -> Response | None:
"""Given a raw Response, unpacks to special values or Exceptions.
The result of this call is what should be passed to users.
@@ -259,7 +273,9 @@ class MessageSender:
raise CleanError(raw_response.error_message)
# Everything else gets lumped in as a remote error.
- raise RemoteError(raw_response.error_message)
+ raise RemoteError(raw_response.error_message,
+ peer_desc=('peer' if self._peer_desc_call is None
+ else self._peer_desc_call(bound_obj)))
assert isinstance(raw_response, Response)
return raw_response
@@ -309,5 +325,6 @@ class BoundMessageSender:
self, message: Message,
raw_response: Response | SysResponse) -> Response | None:
"""Split send (part 2 of 2)."""
- return self._sender.send_split_part_2(message=message,
+ return self._sender.send_split_part_2(bound_obj=self._obj,
+ message=message,
raw_response=raw_response)
diff --git a/tools/efro/rpc.py b/tools/efro/rpc.py
index 93c78ab5..b7a14d03 100644
--- a/tools/efro/rpc.py
+++ b/tools/efro/rpc.py
@@ -61,6 +61,59 @@ class _PeerInfo:
OUR_PROTOCOL = 2
+def ssl_stream_writer_underlying_transport_info(
+ writer: asyncio.StreamWriter) -> str:
+ """For debugging SSL Stream connections; returns raw transport info."""
+ # Note: accessing internals here so just returning info and not
+ # actual objs to reduce potential for breakage.
+ transport = getattr(writer, '_transport', None)
+ if transport is not None:
+ sslproto = getattr(transport, '_ssl_protocol', None)
+ if sslproto is not None:
+ raw_transport = getattr(sslproto, '_transport', None)
+ if raw_transport is not None:
+ return str(raw_transport)
+ return '(not found)'
+
+
+def ssl_stream_writer_force_close_check(writer: asyncio.StreamWriter) -> None:
+ """Ensure a writer is closed; hacky workaround for odd hang."""
+ from efro.call import tpartial
+ from threading import Thread
+ # Hopefully can remove this in Python 3.11?...
+ # see issue with is_closing() below for more details.
+ transport = getattr(writer, '_transport', None)
+ if transport is not None:
+ sslproto = getattr(transport, '_ssl_protocol', None)
+ if sslproto is not None:
+ raw_transport = getattr(sslproto, '_transport', None)
+ if raw_transport is not None:
+ Thread(
+ target=tpartial(
+ _do_writer_force_close_check,
+ weakref.ref(raw_transport),
+ ),
+ daemon=True,
+ ).start()
+
+
+def _do_writer_force_close_check(transport_weak: weakref.ref) -> None:
+ try:
+ # Attempt to bail as soon as the obj dies.
+ # If it hasn't done so by our timeout, force-kill it.
+ starttime = time.monotonic()
+ while time.monotonic() - starttime < 10.0:
+ time.sleep(0.1)
+ if transport_weak() is None:
+ return
+ transport = transport_weak()
+ if transport is not None:
+ logging.info('Forcing abort on stuck transport %s.', transport)
+ transport.abort()
+ except Exception:
+ logging.warning('Error in writer-force-close-check', exc_info=True)
+
+
class _InFlightMessage:
"""Represents a message that is out on the wire."""
@@ -138,6 +191,9 @@ class RPCEndpoint:
self._peer_info: _PeerInfo | None = None
self._keepalive_interval = keepalive_interval
self._keepalive_timeout = keepalive_timeout
+ self._did_close_writer = False
+ self._did_wait_closed_writer = False
+ self._did_out_packets_buildup_warning = False
# Need to hold weak-refs to these otherwise it creates dep-loops
# which keeps us alive.
@@ -156,11 +212,39 @@ class RPCEndpoint:
self._debug_print_call(
f'{self._label}: connected to {peername} at {self._tm()}.')
+ def __del__(self) -> None:
+ if self._run_called:
+ if not self._did_close_writer:
+ logging.warning(
+ 'RPCEndpoint %d dying with run'
+ ' called but writer not closed (transport=%s).', id(self),
+ ssl_stream_writer_underlying_transport_info(self._writer))
+ elif not self._did_wait_closed_writer:
+ logging.warning(
+ 'RPCEndpoint %d dying with run called'
+ ' but writer not wait-closed (transport=%s).', id(self),
+ ssl_stream_writer_underlying_transport_info(self._writer))
+
+ # Currently seeing rare issue where sockets don't go down;
+ # let's add a timer to force the issue until we can figure it out.
+ ssl_stream_writer_force_close_check(self._writer)
+
async def run(self) -> None:
"""Run the endpoint until the connection is lost or closed.
Handles closing the provided reader/writer on close.
"""
+ try:
+ await self._do_run()
+ except asyncio.CancelledError:
+ # We aren't really designed to be cancelled so let's warn
+ # if it happens.
+ logging.warning('RPCEndpoint.run got CancelledError;'
+ ' want to try and avoid this.')
+ raise
+
+ async def _do_run(self) -> None:
+
self._check_env()
if self._run_called:
@@ -186,9 +270,13 @@ class RPCEndpoint:
# We want to know if any errors happened aside from CancelledError
# (which are BaseExceptions, not Exception).
if isinstance(result, Exception):
- if self._debug_print:
- logging.error('Got unexpected error from %s core task: %s',
- self._label, result)
+ logging.warning('Got unexpected error from %s core task: %s',
+ self._label, result)
+
+ if not all(task.done() for task in core_tasks):
+ logging.warning(
+ 'RPCEndpoint %d: not all core tasks marked done after gather.',
+ id(self))
# Shut ourself down.
try:
@@ -228,6 +316,9 @@ class RPCEndpoint:
message_id = self._next_message_id
self._next_message_id = (self._next_message_id + 1) % 65536
+ # FIXME - should handle backpressure (waiting here if there are
+ # enough packets already enqueued).
+
if len(message) > 65535:
# Payload consists of type (1b), message_id (2b),
# len (4b), and data.
@@ -261,6 +352,9 @@ class RPCEndpoint:
try:
return await asyncio.wait_for(msgobj.wait_task, timeout=timeout)
except asyncio.CancelledError as exc:
+ # Question: we assume this means the above wait_for() was
+ # cancelled; what happens if a task running *us* is cancelled
+ # though?
if self._debug_print:
self._debug_print_call(
f'{self._label}: message {message_id} was cancelled.')
@@ -297,9 +391,12 @@ class RPCEndpoint:
for task in self._get_live_tasks():
task.cancel()
+ # Close our writer.
+ assert not self._did_close_writer
if self._debug_print:
self._debug_print_call(f'{self._label}: closing writer...')
self._writer.close()
+ self._did_close_writer = True
# We don't need this anymore and it is likely to be creating a
# dependency loop.
@@ -311,6 +408,7 @@ class RPCEndpoint:
async def wait_closed(self) -> None:
"""I said seagulls; mmmm; stop it now."""
+ # pylint: disable=too-many-branches
self._check_env()
# Make sure we only *enter* this call once.
@@ -321,6 +419,10 @@ class RPCEndpoint:
if not self._closing:
raise RuntimeError('Must be called after close()')
+ if not self._did_close_writer:
+ logging.warning('RPCEndpoint wait_closed() called but never'
+ ' explicitly closed writer.')
+
live_tasks = self._get_live_tasks()
if self._debug_print:
self._debug_print_call(
@@ -333,10 +435,13 @@ class RPCEndpoint:
# We want to know if any errors happened aside from CancelledError
# (which are BaseExceptions, not Exception).
if isinstance(result, Exception):
- if self._debug_print:
- logging.error(
- 'Got unexpected error cleaning up %s task: %s',
- self._label, result)
+ logging.warning('Got unexpected error cleaning up %s task: %s',
+ self._label, result)
+
+ if not all(task.done() for task in live_tasks):
+ logging.warning(
+ 'RPCEndpoint %d: not all live tasks marked done after gather.',
+ id(self))
if self._debug_print:
self._debug_print_call(
@@ -354,10 +459,12 @@ class RPCEndpoint:
# indefinitely. See https://github.com/python/cpython/issues/83939
# It sounds like this should be fixed in 3.11 but for now just
# forcing the issue with a timeout here.
- await asyncio.wait_for(self._writer.wait_closed(), timeout=10.0)
+ await asyncio.wait_for(self._writer.wait_closed(), timeout=30.0)
except asyncio.TimeoutError:
- logging.info('Timeout on _writer.wait_closed() for %s.',
- self._label)
+ logging.info(
+ 'Timeout on _writer.wait_closed() for %s rpc (transport=%s).',
+ self._label,
+ ssl_stream_writer_underlying_transport_info(self._writer))
if self._debug_print:
self._debug_print_call(
f'{self._label}: got timeout in _writer.wait_closed();'
@@ -370,6 +477,12 @@ class RPCEndpoint:
self._debug_print_call(
f'{self._label}: silently ignoring error in'
f' _writer.wait_closed(): {exc}.')
+ except asyncio.CancelledError:
+ logging.warning('RPCEndpoint.wait_closed()'
+ ' got asyncio.CancelledError; not expected.')
+ raise
+ assert not self._did_wait_closed_writer
+ self._did_wait_closed_writer = True
def _tm(self) -> str:
"""Simple readable time value for debugging."""
@@ -494,7 +607,21 @@ class RPCEndpoint:
self._have_out_packets.clear()
self._writer.write(data)
- # await self._writer.drain()
+
+ # This should keep our writer from buffering huge amounts
+ # of outgoing data. We must remember though that we also
+ # need to prevent _out_packets from growing too large and
+ # that part's on us.
+ await self._writer.drain()
+
+ # For now we're not applying backpressure, but let's make
+ # noise if this gets out of hand.
+ if len(self._out_packets) > 200:
+ if not self._did_out_packets_buildup_warning:
+ logging.warning(
+ '_out_packets building up too'
+ ' much on RPCEndpoint %s.', id(self))
+ self._did_out_packets_buildup_warning = True
async def _run_keepalive_task(self) -> None:
"""Send periodic keepalive packets."""
diff --git a/tools/efrotools/pybuild.py b/tools/efrotools/pybuild.py
index 2e36910c..72832238 100644
--- a/tools/efrotools/pybuild.py
+++ b/tools/efrotools/pybuild.py
@@ -15,8 +15,8 @@ if TYPE_CHECKING:
# Python version we build here (not necessarily same as we use in repo).
PY_VER = '3.10'
-PY_VER_EXACT_ANDROID = '3.10.5'
-PY_VER_EXACT_APPLE = '3.10.4'
+PY_VER_EXACT_ANDROID = '3.10.7'
+PY_VER_EXACT_APPLE = '3.10.7'
ANDROID_PYTHON_REPO = 'https://github.com/GRRedWings/python3-android'
@@ -80,18 +80,18 @@ def build_apple(arch: str, debug: bool = False) -> None:
# Customize our minimum version requirements
txt = replace_exact(
txt,
- 'CFLAGS-macOS=-mmacosx-version-min=10.15\n',
- 'CFLAGS-macOS=-mmacosx-version-min=10.15\n',
+ 'VERSION_MIN-macOS=10.15\n',
+ 'VERSION_MIN-macOS=10.15\n',
)
txt = replace_exact(
txt,
- 'CFLAGS-iOS=-mios-version-min=12.0 ',
- 'CFLAGS-iOS=-mios-version-min=12.0 ',
+ 'VERSION_MIN-iOS=12.0\n',
+ 'VERSION_MIN-iOS=12.0\n',
)
txt = replace_exact(
txt,
- 'CFLAGS-tvOS=-mtvos-version-min=9.0 ',
- 'CFLAGS-tvOS=-mtvos-version-min=9.0 ',
+ 'VERSION_MIN-tvOS=9.0\n',
+ 'VERSION_MIN-tvOS=9.0\n',
)
assert '--with-pydebug' not in txt
@@ -119,23 +119,30 @@ def build_apple(arch: str, debug: bool = False) -> None:
# Inject our custom modifications to fire right after their normal
# Setup.local filtering and right before building (and pass the same
# 'slice' value they use so we can use it too).
- txt = replace_exact(
- txt, '\t\t\tsed -e "s/{{slice}}/$$(SLICE-$$(SDK-$(target)))/g" \\\n'
- '\t\t\t> $$(PYTHON_DIR-$(target))/Modules/Setup.local\n',
- '\t\t\tsed -e "s/{{slice}}/$$(SLICE-$$(SDK-$(target)))/g" \\\n'
- '\t\t\t> $$(PYTHON_DIR-$(target))/Modules/Setup.local\n'
- '\tcd $$(PYTHON_DIR-$(target)) && '
- f'../../../../../tools/pcommand python_apple_patch {arch} '
- '"$$(SLICE-$$(SDK-$(target)))"\n')
- txt = replace_exact(
- txt, '\t\t\tsed -e "s/{{slice}}/$$(SLICE-macosx)/g" \\\n'
- '\t\t\t> $$(PYTHON_DIR-$(os))/Modules/Setup.local\n',
- '\t\t\tsed -e "s/{{slice}}/$$(SLICE-macosx)/g" \\\n'
- '\t\t\t> $$(PYTHON_DIR-$(os))/Modules/Setup.local\n'
- '\tcd $$(PYTHON_DIR-$(os)) && '
- f'../../../../../tools/pcommand python_apple_patch {arch} '
- '"$$(SLICE-macosx)"\n')
-
+ # txt = replace_exact(
+ # txt, '\t\t\tsed -e "s/{{slice}}/$$(SLICE-$$(SDK-$(target)))/g" \\\n'
+ # '\t\t\t> $$(PYTHON_DIR-$(target))/Modules/Setup.local\n',
+ # '\t\t\tsed -e "s/{{slice}}/$$(SLICE-$$(SDK-$(target)))/g" \\\n'
+ # '\t\t\t> $$(PYTHON_DIR-$(target))/Modules/Setup.local\n'
+ # '\tcd $$(PYTHON_DIR-$(target)) && '
+ # f'../../../../../tools/pcommand python_apple_patch {arch} '
+ # '"$$(SLICE-$$(SDK-$(target)))"\n')
+ # txt = replace_exact(
+ # txt, '\t\t\tsed -e "s/{{slice}}/$$(SLICE-macosx)/g" \\\n'
+ # '\t\t\t> $$(PYTHON_DIR-$(os))/Modules/Setup.local\n',
+ # '\t\t\tsed -e "s/{{slice}}/$$(SLICE-macosx)/g" \\\n'
+ # '\t\t\t> $$(PYTHON_DIR-$(os))/Modules/Setup.local\n'
+ # '\tcd $$(PYTHON_DIR-$(os)) && '
+ # f'../../../../../tools/pcommand python_apple_patch {arch} '
+ # '"$$(SLICE-macosx)"\n')
+ # txt = replace_exact(
+ # txt,
+ # ' # Configure target Python\n',
+ # ' # Configure target Python\n'
+ # f'\t../../../../../tools/pcommand python_apple_patch'
+ # f'{arch} wtfslice\n',
+ # count=2,
+ # )
writefile('Makefile', txt)
# Ok; let 'er rip.
@@ -228,8 +235,8 @@ def apple_patch(arch: str, slc: str) -> None:
# blow away all the tweaks that this setup does to Setup.local and
# instead apply our very similar ones directly to Setup, just as we
# do for android.
- with open('Modules/Setup.local', 'w', encoding='utf-8') as outfile:
- outfile.write('# cleared by efrotools build\n')
+ # with open('Modules/Setup.local', 'w', encoding='utf-8') as outfile:
+ # outfile.write('# cleared by efrotools build\n')
_patch_setup_file('apple', arch, slc)
_patch_py_ssl()
@@ -292,6 +299,10 @@ def android_patch_ssl() -> None:
def _patch_py_ssl() -> None:
+ # UPDATE: this is now included in Python as of 3.10.6; woohoo!
+ if bool(True):
+ return
+
# I've tracked down an issue where Python's SSL module
# can spend lots of time in SSL_CTX_set_default_verify_paths()
# while holding the GIL, which hitches the game like crazy.