RYDE-WORK/autoresearch
1
0
Fork 0
mirror of https://github.com/RYDE-WORK/autoresearch.git synced 2026-07-22 05:04:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix NaN loss not caught by fast-fail check

Browse Source
This commit is contained in:
Andrej 2026-03-10 22:31:43 -07:00 committed by GitHub
commit 0be1e4fdf9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
train.py
View File

@ -9,6 +9,7 @@ os.environ["PYTORCH_ALLOC_CONF"] = "expandable_segments:True"
os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = "1" os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = "1"
import gc import gc
import math
import time import time
from dataclasses import dataclass, asdict from dataclasses import dataclass, asdict
@ -565,8 +566,8 @@ while True:
train_loss_f = train_loss.item() train_loss_f = train_loss.item()
# Fast fail: abort if loss is exploding # Fast fail: abort if loss is exploding or NaN
if train_loss_f > 100: if math.isnan(train_loss_f) or train_loss_f > 100:
print("FAIL") print("FAIL")
exit(1) exit(1)