mirror of
https://github.com/RYDE-WORK/lnp_ml.git
synced 2026-07-21 21:22:05 +08:00
feat(cv): 断点续跑 - 主循环跳过已完成的 outer fold
启动时若 fold_dir 已有 test_metrics/best_params/epoch_mean,直接读回结果跳过训练。 仅对主循环生效(precomputed_best_params is None),不影响 repeat 阶段。 应对服务器偶发崩溃,崩溃重启后重跑同一 output-dir 可接续未完成的 fold。
This commit is contained in:
parent
7248148515
commit
07b6b44f50
@ -632,6 +632,21 @@ def _run_single_outer_fold(
|
|||||||
fold_dir.mkdir(parents=True, exist_ok=True)
|
fold_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
full_dataset = LNPDataset(df)
|
full_dataset = LNPDataset(df)
|
||||||
|
# === 断点续跑:已完成的 fold 直接跳过(读回磁盘结果)===
|
||||||
|
_tm = fold_dir / "test_metrics.json"
|
||||||
|
_bp = fold_dir / "best_params.json"
|
||||||
|
_em = fold_dir / "epoch_mean.json"
|
||||||
|
if precomputed_best_params is None and _tm.exists() and _bp.exists() and _em.exists():
|
||||||
|
logger.success(f"[SKIP] Outer fold {outer_fold} already done, loading cached results.")
|
||||||
|
with open(_tm) as _f: _tmd = json.load(_f)
|
||||||
|
with open(_bp) as _f: _bpd = json.load(_f)
|
||||||
|
with open(_em) as _f: _emd = json.load(_f)
|
||||||
|
return {
|
||||||
|
"fold": outer_fold,
|
||||||
|
"best_params": _bpd,
|
||||||
|
"epoch_mean": int(_emd.get("epoch_mean", _emd) if isinstance(_emd, dict) else _emd),
|
||||||
|
"test_metrics": _tmd,
|
||||||
|
}
|
||||||
|
|
||||||
logger.info(f"\n{'='*60}")
|
logger.info(f"\n{'='*60}")
|
||||||
logger.info(f"OUTER FOLD {outer_fold}")
|
logger.info(f"OUTER FOLD {outer_fold}")
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user