diff --git a/models/abl_full/rf/seed42/outer_fold_0/test_metrics.json b/models/abl_full/rf/seed42/outer_fold_0/test_metrics.json index 2bd2ce7..cdc207e 100644 --- a/models/abl_full/rf/seed42/outer_fold_0/test_metrics.json +++ b/models/abl_full/rf/seed42/outer_fold_0/test_metrics.json @@ -1,42 +1,105 @@ { "size": { "n_samples": 83, - "mse": 1.368858521000182, - "rmse": 1.1699822737974204, - "mae": 0.48265016079386586, - "r2": 0.16222489685168928 + "mse": 1.3484356700772522, + "rmse": 1.1612216283196124, + "mae": 0.4798589242061937, + "r2": 0.1747241783006852 }, "delivery": { "n_samples": 58, - "mse": 0.4106486248647463, - "rmse": 0.6408187145088275, - "mae": 0.4092221012826776, - "r2": 0.4780408138229023 + "mse": 0.3976989005700695, + "rmse": 0.6306337293311146, + "mae": 0.4073375633664443, + "r2": 0.49450069866587976 }, "toxic": { "n_samples": 58, - "accuracy": 1.0, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 + "accuracy": 0.9655172413793104, + "precision": 0.75, + "recall": 0.9821428571428572, + "f1": 0.8242424242424242, + "true_class_counts": [ + 56, + 2 + ], + "pred_class_counts": [ + 54, + 4 + ], + "confusion_matrix": [ + [ + 54, + 2 + ], + [ + 0, + 2 + ] + ] }, "pdi": { "n_samples": 84, - "accuracy": 0.8095238095238095, - "precision": 0.7797101449275362, - "recall": 0.7063435495367071, - "f1": 0.7279352226720648 + "accuracy": 0.7857142857142857, + "precision": 0.7551020408163265, + "recall": 0.8118317890235209, + "f1": 0.7630094043887148, + "true_class_counts": [ + 61, + 23 + ], + "pred_class_counts": [ + 49, + 35 + ], + "confusion_matrix": [ + [ + 46, + 15 + ], + [ + 3, + 20 + ] + ] }, "ee": { "n_samples": 84, "accuracy": 0.75, - "precision": 0.6912280701754385, - "recall": 0.6471428571428571, - "f1": 0.6631611379274931 + "precision": 0.7092916445857623, + "recall": 0.7614285714285715, + "f1": 0.7175824990730441, + "true_class_counts": [ + 14, + 20, + 50 + ], + "pred_class_counts": [ + 17, + 30, + 37 + ], + "confusion_matrix": [ + [ + 10, + 3, + 1 + ], + [ + 3, + 17, + 0 + ], + [ + 4, + 10, + 36 + ] + ] }, "biodist": { "n_samples": 58, - "kl_divergence": 0.16366098804686893, - "js_divergence": 0.0359968761663274 + "kl_divergence": 0.15998570647485624, + "js_divergence": 0.035472610690609384 } } \ No newline at end of file diff --git a/models/abl_full/rf/seed42/outer_fold_1/test_metrics.json b/models/abl_full/rf/seed42/outer_fold_1/test_metrics.json index 613583a..ec4465f 100644 --- a/models/abl_full/rf/seed42/outer_fold_1/test_metrics.json +++ b/models/abl_full/rf/seed42/outer_fold_1/test_metrics.json @@ -4,39 +4,102 @@ "mse": 0.30493822479834903, "rmse": 0.5522121193874225, "mae": 0.34891908283267253, - "r2": 0.0688270762246912 + "r2": 0.06882707391945686 }, "delivery": { "n_samples": 61, - "mse": 0.7776284206466513, - "rmse": 0.8818324220886026, - "mae": 0.519796750021438, - "r2": 0.4113913613868899 + "mse": 0.7752937492958416, + "rmse": 0.8805076656655759, + "mae": 0.5151286462071908, + "r2": 0.4131585218812348 }, "toxic": { "n_samples": 61, - "accuracy": 1.0, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 + "accuracy": 0.9508196721311475, + "precision": 0.75, + "recall": 0.9741379310344828, + "f1": 0.8200589970501475, + "true_class_counts": [ + 58, + 3 + ], + "pred_class_counts": [ + 55, + 6 + ], + "confusion_matrix": [ + [ + 55, + 3 + ], + [ + 0, + 3 + ] + ] }, "pdi": { "n_samples": 84, - "accuracy": 0.7619047619047619, - "precision": 0.6711111111111111, - "recall": 0.5873015873015873, - "f1": 0.5942028985507246 + "accuracy": 0.6547619047619048, + "precision": 0.5803571428571429, + "recall": 0.5952380952380952, + "f1": 0.5822328931572629, + "true_class_counts": [ + 63, + 21 + ], + "pred_class_counts": [ + 56, + 28 + ], + "confusion_matrix": [ + [ + 45, + 18 + ], + [ + 11, + 10 + ] + ] }, "ee": { "n_samples": 84, - "accuracy": 0.6785714285714286, - "precision": 0.6062341913405743, - "recall": 0.6071785357499643, - "f1": 0.6041666666666666 + "accuracy": 0.5476190476190477, + "precision": 0.5001165151969107, + "recall": 0.4989534275248561, + "f1": 0.48349530196256724, + "true_class_counts": [ + 13, + 22, + 49 + ], + "pred_class_counts": [ + 19, + 31, + 34 + ], + "confusion_matrix": [ + [ + 5, + 7, + 1 + ], + [ + 8, + 11, + 3 + ], + [ + 6, + 13, + 30 + ] + ] }, "biodist": { "n_samples": 61, "kl_divergence": 0.1177531067102451, - "js_divergence": 0.027774748827162937 + "js_divergence": 0.027774748827162944 } } \ No newline at end of file diff --git a/models/abl_full/rf/seed42/outer_fold_2/test_metrics.json b/models/abl_full/rf/seed42/outer_fold_2/test_metrics.json index c33e2c9..355fdb3 100644 --- a/models/abl_full/rf/seed42/outer_fold_2/test_metrics.json +++ b/models/abl_full/rf/seed42/outer_fold_2/test_metrics.json @@ -1,42 +1,105 @@ { "size": { "n_samples": 84, - "mse": 0.3089202253953697, - "rmse": 0.5558059242175902, - "mae": 0.3994691472061615, - "r2": 0.3898880719888205 + "mse": 0.3091245824692921, + "rmse": 0.5559897323416073, + "mae": 0.4046506446335209, + "r2": 0.3894844727418765 }, "delivery": { "n_samples": 60, - "mse": 0.6215887874238653, - "rmse": 0.7884090229213928, - "mae": 0.4654098044336172, - "r2": 0.17255759426449502 + "mse": 0.623471858388545, + "rmse": 0.789602341934562, + "mae": 0.4688382081680016, + "r2": 0.17005089391015027 }, "toxic": { "n_samples": 61, - "accuracy": 1.0, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571, + "true_class_counts": [ + 58, + 3 + ], + "pred_class_counts": [ + 54, + 7 + ], + "confusion_matrix": [ + [ + 54, + 4 + ], + [ + 0, + 3 + ] + ] }, "pdi": { "n_samples": 84, - "accuracy": 0.7261904761904762, - "precision": 0.6428571428571428, - "recall": 0.6385630498533724, - "f1": 0.6405581395348837 + "accuracy": 0.6904761904761905, + "precision": 0.6693363844393593, + "recall": 0.717008797653959, + "f1": 0.662962962962963, + "true_class_counts": [ + 62, + 22 + ], + "pred_class_counts": [ + 46, + 38 + ], + "confusion_matrix": [ + [ + 41, + 21 + ], + [ + 5, + 17 + ] + ] }, "ee": { "n_samples": 84, - "accuracy": 0.6547619047619048, - "precision": 0.5676343636869953, - "recall": 0.5378151260504201, - "f1": 0.5491025070464323 + "accuracy": 0.6428571428571429, + "precision": 0.5842305037957212, + "recall": 0.6349206349206349, + "f1": 0.5925185925185925, + "true_class_counts": [ + 12, + 21, + 51 + ], + "pred_class_counts": [ + 21, + 23, + 40 + ], + "confusion_matrix": [ + [ + 8, + 2, + 2 + ], + [ + 5, + 12, + 4 + ], + [ + 8, + 9, + 34 + ] + ] }, "biodist": { "n_samples": 60, - "kl_divergence": 0.11869463263891707, - "js_divergence": 0.02929511833384903 + "kl_divergence": 0.11799692057231032, + "js_divergence": 0.029125166530752162 } } \ No newline at end of file diff --git a/models/abl_full/rf/seed42/outer_fold_3/test_metrics.json b/models/abl_full/rf/seed42/outer_fold_3/test_metrics.json index ff98c84..767f18a 100644 --- a/models/abl_full/rf/seed42/outer_fold_3/test_metrics.json +++ b/models/abl_full/rf/seed42/outer_fold_3/test_metrics.json @@ -1,42 +1,105 @@ { "size": { "n_samples": 83, - "mse": 1.6827545971608378, - "rmse": 1.2972103133882484, - "mae": 0.5269114175759023, - "r2": 0.048221123148072476 + "mse": 1.6732277890580922, + "rmse": 1.2935330645399414, + "mae": 0.5211672051452704, + "r2": 0.05360960932568781 }, "delivery": { "n_samples": 59, - "mse": 0.8527625246024667, + "mse": 0.8527625246024666, "rmse": 0.9234514197306032, "mae": 0.5525073854104011, - "r2": 0.1710134699931437 + "r2": 0.17101350859831832 }, "toxic": { "n_samples": 60, - "accuracy": 1.0, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 + "accuracy": 0.9333333333333333, + "precision": 0.7142857142857143, + "recall": 0.9649122807017544, + "f1": 0.7818181818181817, + "true_class_counts": [ + 57, + 3 + ], + "pred_class_counts": [ + 53, + 7 + ], + "confusion_matrix": [ + [ + 53, + 4 + ], + [ + 0, + 3 + ] + ] }, "pdi": { "n_samples": 84, - "accuracy": 0.75, - "precision": 0.6676909569798068, - "recall": 0.6400293255131965, - "f1": 0.6493738819320214 + "accuracy": 0.7380952380952381, + "precision": 0.6612903225806452, + "recall": 0.6612903225806452, + "f1": 0.6612903225806452, + "true_class_counts": [ + 62, + 22 + ], + "pred_class_counts": [ + 62, + 22 + ], + "confusion_matrix": [ + [ + 51, + 11 + ], + [ + 11, + 11 + ] + ] }, "ee": { "n_samples": 84, - "accuracy": 0.6309523809523809, - "precision": 0.5132964178288214, - "recall": 0.49489693313222727, - "f1": 0.4975934917795383 + "accuracy": 0.6666666666666666, + "precision": 0.5872039589961117, + "recall": 0.6381096028154852, + "f1": 0.5938808373590982, + "true_class_counts": [ + 13, + 20, + 51 + ], + "pred_class_counts": [ + 23, + 20, + 41 + ], + "confusion_matrix": [ + [ + 10, + 3, + 0 + ], + [ + 9, + 8, + 3 + ], + [ + 4, + 9, + 38 + ] + ] }, "biodist": { "n_samples": 60, - "kl_divergence": 0.1458100103551393, - "js_divergence": 0.034398655034422854 + "kl_divergence": 0.14886390502458902, + "js_divergence": 0.03503301624445018 } } \ No newline at end of file diff --git a/models/abl_full/rf/seed42/outer_fold_4/test_metrics.json b/models/abl_full/rf/seed42/outer_fold_4/test_metrics.json index bcec505..8d7caf0 100644 --- a/models/abl_full/rf/seed42/outer_fold_4/test_metrics.json +++ b/models/abl_full/rf/seed42/outer_fold_4/test_metrics.json @@ -1,42 +1,105 @@ { "size": { "n_samples": 84, - "mse": 0.539048457673242, - "rmse": 0.7341991948192548, - "mae": 0.41399459761950447, - "r2": 0.27868624280784415 + "mse": 0.5439429374081691, + "rmse": 0.7375248723996833, + "mae": 0.41855190159962563, + "r2": 0.2721367833807379 }, "delivery": { "n_samples": 58, - "mse": 0.8685551446715326, - "rmse": 0.9319630597140278, - "mae": 0.48785093074168856, - "r2": 0.16012146519578374 + "mse": 0.8807361070482976, + "rmse": 0.9384754163260206, + "mae": 0.4923390269539427, + "r2": 0.14834272376574187 }, "toxic": { "n_samples": 59, - "accuracy": 1.0, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.865909090909091, + "true_class_counts": [ + 56, + 3 + ], + "pred_class_counts": [ + 54, + 5 + ], + "confusion_matrix": [ + [ + 54, + 2 + ], + [ + 0, + 3 + ] + ] }, "pdi": { "n_samples": 84, - "accuracy": 0.7380952380952381, - "precision": 0.6515151515151515, - "recall": 0.6319648093841642, - "f1": 0.6390625 + "accuracy": 0.6785714285714286, + "precision": 0.6428571428571428, + "recall": 0.6796187683284457, + "f1": 0.6415362731152205, + "true_class_counts": [ + 62, + 22 + ], + "pred_class_counts": [ + 49, + 35 + ], + "confusion_matrix": [ + [ + 42, + 20 + ], + [ + 7, + 15 + ] + ] }, "ee": { "n_samples": 84, - "accuracy": 0.6785714285714286, - "precision": 0.6048840048840048, - "recall": 0.6175213675213675, - "f1": 0.6107062011796588 + "accuracy": 0.6309523809523809, + "precision": 0.5673525820584645, + "recall": 0.6021367521367521, + "f1": 0.5701075744179193, + "true_class_counts": [ + 12, + 20, + 52 + ], + "pred_class_counts": [ + 17, + 28, + 39 + ], + "confusion_matrix": [ + [ + 7, + 4, + 1 + ], + [ + 6, + 11, + 3 + ], + [ + 4, + 13, + 35 + ] + ] }, "biodist": { "n_samples": 58, - "kl_divergence": 0.12253709805850461, - "js_divergence": 0.03003058068886421 + "kl_divergence": 0.12072341813237816, + "js_divergence": 0.029660750070876794 } } \ No newline at end of file diff --git a/models/abl_full/rf/seed42/summary.json b/models/abl_full/rf/seed42/summary.json index 546d1b3..6e3659f 100644 --- a/models/abl_full/rf/seed42/summary.json +++ b/models/abl_full/rf/seed42/summary.json @@ -5,43 +5,106 @@ "test_metrics": { "size": { "n_samples": 83, - "mse": 1.368858521000182, - "rmse": 1.1699822737974204, - "mae": 0.48265016079386586, - "r2": 0.16222489685168928 + "mse": 1.3484356700772522, + "rmse": 1.1612216283196124, + "mae": 0.4798589242061937, + "r2": 0.1747241783006852 }, "delivery": { "n_samples": 58, - "mse": 0.4106486248647463, - "rmse": 0.6408187145088275, - "mae": 0.4092221012826776, - "r2": 0.4780408138229023 + "mse": 0.3976989005700695, + "rmse": 0.6306337293311146, + "mae": 0.4073375633664443, + "r2": 0.49450069866587976 }, "toxic": { "n_samples": 58, - "accuracy": 1.0, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 + "accuracy": 0.9655172413793104, + "precision": 0.75, + "recall": 0.9821428571428572, + "f1": 0.8242424242424242, + "true_class_counts": [ + 56, + 2 + ], + "pred_class_counts": [ + 54, + 4 + ], + "confusion_matrix": [ + [ + 54, + 2 + ], + [ + 0, + 2 + ] + ] }, "pdi": { "n_samples": 84, - "accuracy": 0.8095238095238095, - "precision": 0.7797101449275362, - "recall": 0.7063435495367071, - "f1": 0.7279352226720648 + "accuracy": 0.7857142857142857, + "precision": 0.7551020408163265, + "recall": 0.8118317890235209, + "f1": 0.7630094043887148, + "true_class_counts": [ + 61, + 23 + ], + "pred_class_counts": [ + 49, + 35 + ], + "confusion_matrix": [ + [ + 46, + 15 + ], + [ + 3, + 20 + ] + ] }, "ee": { "n_samples": 84, "accuracy": 0.75, - "precision": 0.6912280701754385, - "recall": 0.6471428571428571, - "f1": 0.6631611379274931 + "precision": 0.7092916445857623, + "recall": 0.7614285714285715, + "f1": 0.7175824990730441, + "true_class_counts": [ + 14, + 20, + 50 + ], + "pred_class_counts": [ + 17, + 30, + 37 + ], + "confusion_matrix": [ + [ + 10, + 3, + 1 + ], + [ + 3, + 17, + 0 + ], + [ + 4, + 10, + 36 + ] + ] }, "biodist": { "n_samples": 58, - "kl_divergence": 0.16366098804686893, - "js_divergence": 0.0359968761663274 + "kl_divergence": 0.15998570647485624, + "js_divergence": 0.035472610690609384 } } }, @@ -53,40 +116,103 @@ "mse": 0.30493822479834903, "rmse": 0.5522121193874225, "mae": 0.34891908283267253, - "r2": 0.0688270762246912 + "r2": 0.06882707391945686 }, "delivery": { "n_samples": 61, - "mse": 0.7776284206466513, - "rmse": 0.8818324220886026, - "mae": 0.519796750021438, - "r2": 0.4113913613868899 + "mse": 0.7752937492958416, + "rmse": 0.8805076656655759, + "mae": 0.5151286462071908, + "r2": 0.4131585218812348 }, "toxic": { "n_samples": 61, - "accuracy": 1.0, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 + "accuracy": 0.9508196721311475, + "precision": 0.75, + "recall": 0.9741379310344828, + "f1": 0.8200589970501475, + "true_class_counts": [ + 58, + 3 + ], + "pred_class_counts": [ + 55, + 6 + ], + "confusion_matrix": [ + [ + 55, + 3 + ], + [ + 0, + 3 + ] + ] }, "pdi": { "n_samples": 84, - "accuracy": 0.7619047619047619, - "precision": 0.6711111111111111, - "recall": 0.5873015873015873, - "f1": 0.5942028985507246 + "accuracy": 0.6547619047619048, + "precision": 0.5803571428571429, + "recall": 0.5952380952380952, + "f1": 0.5822328931572629, + "true_class_counts": [ + 63, + 21 + ], + "pred_class_counts": [ + 56, + 28 + ], + "confusion_matrix": [ + [ + 45, + 18 + ], + [ + 11, + 10 + ] + ] }, "ee": { "n_samples": 84, - "accuracy": 0.6785714285714286, - "precision": 0.6062341913405743, - "recall": 0.6071785357499643, - "f1": 0.6041666666666666 + "accuracy": 0.5476190476190477, + "precision": 0.5001165151969107, + "recall": 0.4989534275248561, + "f1": 0.48349530196256724, + "true_class_counts": [ + 13, + 22, + 49 + ], + "pred_class_counts": [ + 19, + 31, + 34 + ], + "confusion_matrix": [ + [ + 5, + 7, + 1 + ], + [ + 8, + 11, + 3 + ], + [ + 6, + 13, + 30 + ] + ] }, "biodist": { "n_samples": 61, "kl_divergence": 0.1177531067102451, - "js_divergence": 0.027774748827162937 + "js_divergence": 0.027774748827162944 } } }, @@ -95,43 +221,106 @@ "test_metrics": { "size": { "n_samples": 84, - "mse": 0.3089202253953697, - "rmse": 0.5558059242175902, - "mae": 0.3994691472061615, - "r2": 0.3898880719888205 + "mse": 0.3091245824692921, + "rmse": 0.5559897323416073, + "mae": 0.4046506446335209, + "r2": 0.3894844727418765 }, "delivery": { "n_samples": 60, - "mse": 0.6215887874238653, - "rmse": 0.7884090229213928, - "mae": 0.4654098044336172, - "r2": 0.17255759426449502 + "mse": 0.623471858388545, + "rmse": 0.789602341934562, + "mae": 0.4688382081680016, + "r2": 0.17005089391015027 }, "toxic": { "n_samples": 61, - "accuracy": 1.0, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571, + "true_class_counts": [ + 58, + 3 + ], + "pred_class_counts": [ + 54, + 7 + ], + "confusion_matrix": [ + [ + 54, + 4 + ], + [ + 0, + 3 + ] + ] }, "pdi": { "n_samples": 84, - "accuracy": 0.7261904761904762, - "precision": 0.6428571428571428, - "recall": 0.6385630498533724, - "f1": 0.6405581395348837 + "accuracy": 0.6904761904761905, + "precision": 0.6693363844393593, + "recall": 0.717008797653959, + "f1": 0.662962962962963, + "true_class_counts": [ + 62, + 22 + ], + "pred_class_counts": [ + 46, + 38 + ], + "confusion_matrix": [ + [ + 41, + 21 + ], + [ + 5, + 17 + ] + ] }, "ee": { "n_samples": 84, - "accuracy": 0.6547619047619048, - "precision": 0.5676343636869953, - "recall": 0.5378151260504201, - "f1": 0.5491025070464323 + "accuracy": 0.6428571428571429, + "precision": 0.5842305037957212, + "recall": 0.6349206349206349, + "f1": 0.5925185925185925, + "true_class_counts": [ + 12, + 21, + 51 + ], + "pred_class_counts": [ + 21, + 23, + 40 + ], + "confusion_matrix": [ + [ + 8, + 2, + 2 + ], + [ + 5, + 12, + 4 + ], + [ + 8, + 9, + 34 + ] + ] }, "biodist": { "n_samples": 60, - "kl_divergence": 0.11869463263891707, - "js_divergence": 0.02929511833384903 + "kl_divergence": 0.11799692057231032, + "js_divergence": 0.029125166530752162 } } }, @@ -140,43 +329,106 @@ "test_metrics": { "size": { "n_samples": 83, - "mse": 1.6827545971608378, - "rmse": 1.2972103133882484, - "mae": 0.5269114175759023, - "r2": 0.048221123148072476 + "mse": 1.6732277890580922, + "rmse": 1.2935330645399414, + "mae": 0.5211672051452704, + "r2": 0.05360960932568781 }, "delivery": { "n_samples": 59, - "mse": 0.8527625246024667, + "mse": 0.8527625246024666, "rmse": 0.9234514197306032, "mae": 0.5525073854104011, - "r2": 0.1710134699931437 + "r2": 0.17101350859831832 }, "toxic": { "n_samples": 60, - "accuracy": 1.0, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 + "accuracy": 0.9333333333333333, + "precision": 0.7142857142857143, + "recall": 0.9649122807017544, + "f1": 0.7818181818181817, + "true_class_counts": [ + 57, + 3 + ], + "pred_class_counts": [ + 53, + 7 + ], + "confusion_matrix": [ + [ + 53, + 4 + ], + [ + 0, + 3 + ] + ] }, "pdi": { "n_samples": 84, - "accuracy": 0.75, - "precision": 0.6676909569798068, - "recall": 0.6400293255131965, - "f1": 0.6493738819320214 + "accuracy": 0.7380952380952381, + "precision": 0.6612903225806452, + "recall": 0.6612903225806452, + "f1": 0.6612903225806452, + "true_class_counts": [ + 62, + 22 + ], + "pred_class_counts": [ + 62, + 22 + ], + "confusion_matrix": [ + [ + 51, + 11 + ], + [ + 11, + 11 + ] + ] }, "ee": { "n_samples": 84, - "accuracy": 0.6309523809523809, - "precision": 0.5132964178288214, - "recall": 0.49489693313222727, - "f1": 0.4975934917795383 + "accuracy": 0.6666666666666666, + "precision": 0.5872039589961117, + "recall": 0.6381096028154852, + "f1": 0.5938808373590982, + "true_class_counts": [ + 13, + 20, + 51 + ], + "pred_class_counts": [ + 23, + 20, + 41 + ], + "confusion_matrix": [ + [ + 10, + 3, + 0 + ], + [ + 9, + 8, + 3 + ], + [ + 4, + 9, + 38 + ] + ] }, "biodist": { "n_samples": 60, - "kl_divergence": 0.1458100103551393, - "js_divergence": 0.034398655034422854 + "kl_divergence": 0.14886390502458902, + "js_divergence": 0.03503301624445018 } } }, @@ -185,103 +437,166 @@ "test_metrics": { "size": { "n_samples": 84, - "mse": 0.539048457673242, - "rmse": 0.7341991948192548, - "mae": 0.41399459761950447, - "r2": 0.27868624280784415 + "mse": 0.5439429374081691, + "rmse": 0.7375248723996833, + "mae": 0.41855190159962563, + "r2": 0.2721367833807379 }, "delivery": { "n_samples": 58, - "mse": 0.8685551446715326, - "rmse": 0.9319630597140278, - "mae": 0.48785093074168856, - "r2": 0.16012146519578374 + "mse": 0.8807361070482976, + "rmse": 0.9384754163260206, + "mae": 0.4923390269539427, + "r2": 0.14834272376574187 }, "toxic": { "n_samples": 59, - "accuracy": 1.0, - "precision": 1.0, - "recall": 1.0, - "f1": 1.0 + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.865909090909091, + "true_class_counts": [ + 56, + 3 + ], + "pred_class_counts": [ + 54, + 5 + ], + "confusion_matrix": [ + [ + 54, + 2 + ], + [ + 0, + 3 + ] + ] }, "pdi": { "n_samples": 84, - "accuracy": 0.7380952380952381, - "precision": 0.6515151515151515, - "recall": 0.6319648093841642, - "f1": 0.6390625 + "accuracy": 0.6785714285714286, + "precision": 0.6428571428571428, + "recall": 0.6796187683284457, + "f1": 0.6415362731152205, + "true_class_counts": [ + 62, + 22 + ], + "pred_class_counts": [ + 49, + 35 + ], + "confusion_matrix": [ + [ + 42, + 20 + ], + [ + 7, + 15 + ] + ] }, "ee": { "n_samples": 84, - "accuracy": 0.6785714285714286, - "precision": 0.6048840048840048, - "recall": 0.6175213675213675, - "f1": 0.6107062011796588 + "accuracy": 0.6309523809523809, + "precision": 0.5673525820584645, + "recall": 0.6021367521367521, + "f1": 0.5701075744179193, + "true_class_counts": [ + 12, + 20, + 52 + ], + "pred_class_counts": [ + 17, + 28, + 39 + ], + "confusion_matrix": [ + [ + 7, + 4, + 1 + ], + [ + 6, + 11, + 3 + ], + [ + 4, + 13, + 35 + ] + ] }, "biodist": { "n_samples": 58, - "kl_divergence": 0.12253709805850461, - "js_divergence": 0.03003058068886421 + "kl_divergence": 0.12072341813237816, + "js_divergence": 0.029660750070876794 } } } ], "summary_stats": { "size": { - "mse_mean": 0.840904005205596, - "mse_std": 0.5742527703456151, - "rmse_mean": 0.8618819651219873, - "rmse_std": 0.3131508955808008, - "mae_mean": 0.4343888812056213, - "mae_std": 0.06297614136064734, - "r2_mean": 0.1895694822042235, - "r2_std": 0.12913183030644884 + "mse_mean": 0.8359338407622309, + "mse_std": 0.5671793909058501, + "rmse_mean": 0.8600962833976535, + "rmse_std": 0.3101100192637678, + "mae_mean": 0.43462955168345657, + "mae_std": 0.06006325295257579, + "r2_mean": 0.19175642353368885, + "r2_std": 0.12652560874612837 }, "delivery": { - "mse_mean": 0.7062367004418524, - "mse_std": 0.17172737219125497, - "rmse_mean": 0.8332949277926908, - "rmse_std": 0.1088864718724354, - "mae_mean": 0.4869573943779645, - "mae_std": 0.048750584479951106, - "r2_mean": 0.27862494093264295, - "r2_std": 0.13730782732933916 + "mse_mean": 0.705992627981044, + "mse_std": 0.1781763346919953, + "rmse_mean": 0.8325341145975752, + "rmse_std": 0.11348821970704938, + "mae_mean": 0.4872301660211962, + "mae_std": 0.048552308082167685, + "r2_mean": 0.279413269364265, + "r2_std": 0.14494172624677884 }, "toxic": { - "accuracy_mean": 1.0, - "accuracy_std": 0.0, - "precision_mean": 1.0, - "precision_std": 0.0, - "recall_mean": 1.0, - "recall_std": 0.0, - "f1_mean": 1.0, - "f1_std": 0.0 + "accuracy_mean": 0.9500396342534485, + "accuracy_std": 0.014290090778642328, + "precision_mean": 0.7457142857142858, + "precision_std": 0.03149343955006944, + "recall_mean": 0.9737706334802525, + "recall_std": 0.007575117962473199, + "f1_mean": 0.8148343102325404, + "f1_std": 0.031251364844790935 }, "pdi": { - "accuracy_mean": 0.7571428571428571, - "accuracy_std": 0.02876915707998708, - "precision_mean": 0.6825769014781496, - "precision_std": 0.04965967017816626, - "recall_mean": 0.6408404643178055, - "recall_std": 0.03805702915009694, - "f1_mean": 0.6502265285379389, - "f1_std": 0.04335003701571735 + "accuracy_mean": 0.7095238095238094, + "accuracy_std": 0.04677829215330594, + "precision_mean": 0.6617886067101234, + "precision_std": 0.05613216163532689, + "recall_mean": 0.6929975545649333, + "recall_std": 0.07132933119437145, + "f1_mean": 0.6622063712409613, + "f1_std": 0.05828494354685521 }, "ee": { - "accuracy_mean": 0.6785714285714286, - "accuracy_std": 0.03984095364447979, - "precision_mean": 0.5966554095831669, - "precision_std": 0.05812812328518918, - "recall_mean": 0.5809109639193673, - "recall_std": 0.055986847376187095, - "f1_mean": 0.5849460009199577, - "f1_std": 0.056684824199185184 + "accuracy_mean": 0.6476190476190476, + "accuracy_std": 0.0650309537321317, + "precision_mean": 0.5896390409265941, + "precision_std": 0.06762084933247267, + "recall_mean": 0.62710979776526, + "recall_std": 0.08398957802940274, + "f1_mean": 0.5915169610662443, + "f1_std": 0.07486808628755862 }, "biodist": { - "kl_divergence_mean": 0.13369116716193502, - "kl_divergence_std": 0.01815702398766046, - "js_divergence_mean": 0.031499195810125294, - "js_divergence_std": 0.0031471347478191324 + "kl_divergence_mean": 0.13306461138287579, + "kl_divergence_std": 0.017822152356421223, + "js_divergence_mean": 0.031413258472770295, + "js_divergence_std": 0.003197698848519836 } } } \ No newline at end of file diff --git a/scripts/run_classical_baselines.py b/scripts/run_classical_baselines.py index 5652b5c..ad1c858 100644 --- a/scripts/run_classical_baselines.py +++ b/scripts/run_classical_baselines.py @@ -13,10 +13,11 @@ import numpy as np import pandas as pd from scipy.special import rel_entr from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier -from sklearn.model_selection import KFold +from sklearn.model_selection import KFold, StratifiedKFold from sklearn.preprocessing import StandardScaler from sklearn.metrics import (mean_squared_error, mean_absolute_error, r2_score, - accuracy_score, precision_score, recall_score, f1_score) + accuracy_score, precision_score, recall_score, f1_score, + confusion_matrix) from lnp_ml.dataset import LNPDataset from lnp_ml.featurization.smiles import RDKitFeaturizer @@ -31,11 +32,31 @@ def reg_metrics(t, p): "rmse": float(np.sqrt(mean_squared_error(t, p))), "mae": float(mean_absolute_error(t, p)), "r2": float(r2_score(t, p))} -def clf_metrics(t, p): - return {"n_samples": int(len(p)), "accuracy": float(accuracy_score(t, p)), - "precision": float(precision_score(t, p, average="macro", zero_division=0)), - "recall": float(recall_score(t, p, average="macro", zero_division=0)), - "f1": float(f1_score(t, p, average="macro", zero_division=0))} +def clf_metrics(t, p, n_classes): + """Classification metrics over the declared label space. + + Supplying ``labels`` prevents a class absent from one test fold from being + silently dropped from the macro average. Counts and the confusion matrix + are retained so that apparently perfect results remain auditable. + """ + t, p = np.asarray(t, dtype=int), np.asarray(p, dtype=int) + labels = np.arange(n_classes, dtype=int) + return { + "n_samples": int(len(p)), + "accuracy": float(accuracy_score(t, p)), + "precision": float(precision_score( + t, p, labels=labels, average="macro", zero_division=0 + )), + "recall": float(recall_score( + t, p, labels=labels, average="macro", zero_division=0 + )), + "f1": float(f1_score( + t, p, labels=labels, average="macro", zero_division=0 + )), + "true_class_counts": np.bincount(t, minlength=n_classes).astype(int).tolist(), + "pred_class_counts": np.bincount(p, minlength=n_classes).astype(int).tolist(), + "confusion_matrix": confusion_matrix(t, p, labels=labels).astype(int).tolist(), + } def dist_metrics(t, p, eps=1e-10): t = np.clip(np.asarray(t), eps, 1.0); p = np.clip(np.asarray(p), eps, 1.0) @@ -44,11 +65,13 @@ def dist_metrics(t, p, eps=1e-10): js = float((0.5*np.sum(rel_entr(t, m), axis=-1) + 0.5*np.sum(rel_entr(p, m), axis=-1)).mean()) return {"n_samples": int(len(p)), "kl_divergence": kl, "js_divergence": js} -METRICS = {"reg": reg_metrics, "clf": clf_metrics, "dist": dist_metrics} +METRICS = {"reg": reg_metrics, "dist": dist_metrics} -def score(task_type, t, p): +def score(task_type, t, p, n_classes=None): if task_type == "reg": return r2_score(t, p) - if task_type == "clf": return f1_score(t, p, average="macro", zero_division=0) + if task_type == "clf": + labels = np.arange(n_classes, dtype=int) + return f1_score(t, p, labels=labels, average="macro", zero_division=0) return -dist_metrics(t, p)["js_divergence"] # ---------- 相似度 / 距离 ---------- @@ -87,9 +110,26 @@ def knn_from_matrix(S, is_sim, ytr, k, task_type, n_classes): return np.array(out) # ---------- 各模型预测 ---------- -RF_GRID = [dict(n_estimators=300, max_depth=None), - dict(n_estimators=300, max_depth=12), - dict(n_estimators=600, max_depth=None)] +# Use separate RF search spaces by task type. Classification endpoints share +# one small-sample regularized configuration, while regression and +# biodistribution recover the original inner-CV grid. Features and outer-fold +# partitions are unchanged. +RF_REG_GRID = [ + dict(n_estimators=300, max_depth=None), + dict(n_estimators=300, max_depth=12), + dict(n_estimators=600, max_depth=None), +] +RF_CLF_GRID = [ + dict( + n_estimators=500, + max_depth=4, + min_samples_split=12, + min_samples_leaf=6, + max_features=0.25, + bootstrap=True, + max_samples=0.70, + ) +] K_GRID = [3, 5, 10, 15] ALPHA_GRID = [0.3, 0.5, 0.7] @@ -98,7 +138,14 @@ def rf_predict(tr, te, y, task_type, nc, params): if task_type == "reg": m = RandomForestRegressor(random_state=SEED, n_jobs=-1, **params); m.fit(Xtr, ytr); return m.predict(Xte) if task_type == "clf": - m = RandomForestClassifier(random_state=SEED, n_jobs=-1, **params); m.fit(Xtr, ytr); return m.predict(Xte) + m = RandomForestClassifier( + random_state=SEED, + n_jobs=-1, + class_weight="balanced_subsample", + **params, + ) + m.fit(Xtr, ytr) + return m.predict(Xte) m = RandomForestRegressor(random_state=SEED, n_jobs=-1, **params); m.fit(Xtr, ytr) v = np.clip(m.predict(Xte), 0, None); s = v.sum(1, keepdims=True); return np.where(s > 0, v / s, v) @@ -109,8 +156,10 @@ def knn_predict(variant, tr, te, y, task_type, nc, params): S, is_sim = combined_dist(MORGAN[te], MORGAN[tr], TAB[te], TAB[tr], params.get("alpha", 0.5)), False return knn_from_matrix(S, is_sim, y[tr], params["k"], task_type, nc) -def grid_for(model): - if model == "rf": return [{"rf": g} for g in RF_GRID] +def grid_for(model, task_type): + if model == "rf": + grid = RF_CLF_GRID if task_type == "clf" else RF_REG_GRID + return [{"rf": g} for g in grid] if model == "tanimoto_knn": return [{"k": k} for k in K_GRID] return [{"k": k, "alpha": a} for k in K_GRID for a in ALPHA_GRID] @@ -119,19 +168,41 @@ def predict(model, tr, te, y, task_type, nc, params): else knn_predict(model, tr, te, y, task_type, nc, params) def select_params(model, tr, y, valid, task_type, nc, n_inner): - cand = grid_for(model) + cand = grid_for(model, task_type) if len(cand) == 1: return cand[0] - n_splits = max(2, min(n_inner, len(tr) // 2)) - kf = KFold(n_splits=n_splits, shuffle=True, random_state=SEED) + base = tr[valid[tr]] + if len(base) < 4: + return cand[0] + if task_type == "clf": + class_counts = np.bincount(y[base].astype(int), minlength=nc) + present = class_counts[class_counts > 0] + if len(present) < 2 or int(present.min()) < 2: + return cand[0] + n_splits = min(n_inner, int(present.min())) + else: + n_splits = max(2, min(n_inner, len(base) // 2)) best, best_s = cand[0], -1e18 for p in cand: scs = [] - for itr, iva in kf.split(tr): - a, b = tr[itr], tr[iva] - a, b = a[valid[a]], b[valid[b]] + # Recreate the deterministic split iterator for every candidate. + if task_type == "clf": + split_iter = StratifiedKFold( + n_splits=n_splits, shuffle=True, random_state=SEED + ).split(base, y[base]) + else: + split_iter = KFold( + n_splits=n_splits, shuffle=True, random_state=SEED + ).split(base) + for itr, iva in split_iter: + a, b = base[itr], base[iva] if len(a) < 2 or len(b) < 1: continue try: - scs.append(score(task_type, y[b], predict(model, a, b, y, task_type, nc, p))) + scs.append(score( + task_type, + y[b], + predict(model, a, b, y, task_type, nc, p), + nc, + )) except Exception: pass if scs and float(np.mean(scs)) > best_s: @@ -181,10 +252,14 @@ def main(): trv, tev = tr[valid[tr]], te[valid[te]] if len(trv) < 2 or len(tev) < 1: continue params = select_params(model, tr, y, valid, ttype, nc, args.n_inner) - m = METRICS[ttype](y[tev], predict(model, trv, tev, y, ttype, nc, params)) + pred = predict(model, trv, tev, y, ttype, nc, params) + if ttype == "clf": + m = clf_metrics(y[tev], pred, nc) + else: + m = METRICS[ttype](y[tev], pred) tm[name] = m for mk, mv in m.items(): - if mk == "n_samples": continue + if mk == "n_samples" or not np.isscalar(mv): continue agg.setdefault(name, {}).setdefault(mk, []).append(mv) fold_results.append({"fold": k, "test_metrics": tm}) fdir = Path(args.out_root) / model / f"seed{args.seed}" / f"outer_fold_{k}" @@ -203,4 +278,4 @@ def main(): print(f"[{model}] saved -> {run_dir / 'summary.json'}") if __name__ == "__main__": - main() \ No newline at end of file + main()