import json, glob, os
import numpy as np
from scipy.stats import ttest_rel, wilcoxon

A_DIR = "models/benchmark_cmp/mpnn"
B_DIR = "models/benchmark_cmp/chemeleon"
# (metric_key, higher_better)
METRICS = [("best_val_rmse", False), ("best_val_r2", True), ("best_val_loss", False)]


def collect(base):
    """返回 {(seed, fold_idx): {metric: value}}"""
    out = {}
    for f in glob.glob(os.path.join(base, "seed*", "cv_results.json")):
        seed = os.path.basename(os.path.dirname(f))
        with open(f) as fh:
            d = json.load(fh)
        for r in d["fold_results"]:
            out[(seed, r["fold_idx"])] = r
    return out


a, b = collect(A_DIR), collect(B_DIR)
keys = sorted(set(a) & set(b))
print(f"配对样本数 n = {len(keys)}  (seed × fold)\n")
for m, hb in METRICS:
    da = np.array([a[k][m] for k in keys])
    db = np.array([b[k][m] for k in keys])
    diff = (db - da) if hb else (da - db)  # 正 = CheMeleon 更好
    tp = ttest_rel(db, da).pvalue
    try:
        wp = wilcoxon(db, da).pvalue
    except ValueError:
        wp = float("nan")
    print(f"{m:16s}  MPNN={da.mean():.4f}±{da.std():.4f}  "
          f"CheMeleon={db.mean():.4f}±{db.std():.4f}  "
          f"Δ(Chem优)={diff.mean():+.4f}  Wilcoxon p={wp:.3f}  t p={tp:.3f}")