mirror of
https://github.com/RYDE-WORK/lnp_ml.git
synced 2026-09-18 21:13:20 +08:00
37 lines
1.2 KiB
Python
37 lines
1.2 KiB
Python
import json, glob, os
|
||
import numpy as np
|
||
from scipy.stats import ttest_rel, wilcoxon
|
||
|
||
A_DIR = "models/benchmark_cmp/mpnn"
|
||
B_DIR = "models/benchmark_cmp/chemeleon"
|
||
# (metric_key, higher_better)
|
||
METRICS = [("best_val_rmse", False), ("best_val_r2", True), ("best_val_loss", False)]
|
||
|
||
|
||
def collect(base):
|
||
"""返回 {(seed, fold_idx): {metric: value}}"""
|
||
out = {}
|
||
for f in glob.glob(os.path.join(base, "seed*", "cv_results.json")):
|
||
seed = os.path.basename(os.path.dirname(f))
|
||
with open(f) as fh:
|
||
d = json.load(fh)
|
||
for r in d["fold_results"]:
|
||
out[(seed, r["fold_idx"])] = r
|
||
return out
|
||
|
||
|
||
a, b = collect(A_DIR), collect(B_DIR)
|
||
keys = sorted(set(a) & set(b))
|
||
print(f"配对样本数 n = {len(keys)} (seed × fold)\n")
|
||
for m, hb in METRICS:
|
||
da = np.array([a[k][m] for k in keys])
|
||
db = np.array([b[k][m] for k in keys])
|
||
diff = (db - da) if hb else (da - db) # 正 = CheMeleon 更好
|
||
tp = ttest_rel(db, da).pvalue
|
||
try:
|
||
wp = wilcoxon(db, da).pvalue
|
||
except ValueError:
|
||
wp = float("nan")
|
||
print(f"{m:16s} MPNN={da.mean():.4f}±{da.std():.4f} "
|
||
f"CheMeleon={db.mean():.4f}±{db.std():.4f} "
|
||
f"Δ(Chem优)={diff.mean():+.4f} Wilcoxon p={wp:.3f} t p={tp:.3f}") |