From 67db7fa4737edf32fcf8e0bc7c086f88455b9dbc Mon Sep 17 00:00:00 2001
From: Dicong Li <DicongLi@users.noreply.github.com>
Date: Fri, 14 Aug 2026 17:37:31 +0000
Subject: [PATCH] Add pretrained encoder baseline scripts and results

---
 .../baseline_pretrain_s42_summary.json        | 932 +++++++++++++++++
 .../graphmvp_pretrain_s42_summary.json        | 932 +++++++++++++++++
 .../grover_pretrain_s42_summary.json          | 932 +++++++++++++++++
 .../moe_pretrain_s42_summary.json             | 977 ++++++++++++++++++
 .../molformer_pretrain_s42_summary.json       | 932 +++++++++++++++++
 scripts_run/encode_embeddings/README.md       |  43 +
 .../encode_embeddings/encode_graphmvp.py      | 122 +++
 .../encode_embeddings/encode_grover.py        |  57 +
 .../encode_embeddings/encode_molformer.py     |  51 +
 9 files changed, 4978 insertions(+)
 create mode 100644 results/pretrained_encoders/baseline_pretrain_s42_summary.json
 create mode 100644 results/pretrained_encoders/graphmvp_pretrain_s42_summary.json
 create mode 100644 results/pretrained_encoders/grover_pretrain_s42_summary.json
 create mode 100644 results/pretrained_encoders/moe_pretrain_s42_summary.json
 create mode 100644 results/pretrained_encoders/molformer_pretrain_s42_summary.json
 create mode 100644 scripts_run/encode_embeddings/README.md
 create mode 100644 scripts_run/encode_embeddings/encode_graphmvp.py
 create mode 100644 scripts_run/encode_embeddings/encode_grover.py
 create mode 100644 scripts_run/encode_embeddings/encode_molformer.py

diff --git a/results/pretrained_encoders/baseline_pretrain_s42_summary.json b/results/pretrained_encoders/baseline_pretrain_s42_summary.json
new file mode 100644
index 0000000..a68e516
--- /dev/null
+++ b/results/pretrained_encoders/baseline_pretrain_s42_summary.json
@@ -0,0 +1,932 @@
+{
+  "fold_results": [
+    {
+      "fold": 0,
+      "best_params": {
+        "dropout": 0.3980773018180746,
+        "lr": 0.0009896030589860336,
+        "weight_decay": 0.0002988554096805342,
+        "backbone_lr_ratio": 0.2882797738700643,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 14,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.601088294900837,
+          "rmse": 1.265341177272295,
+          "mae": 0.5760945507650634,
+          "r2": 0.02009455288902051
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 0.569556123704086,
+          "rmse": 0.7546894220168228,
+          "mae": 0.5891120558411911,
+          "r2": 0.2760597975244864
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7738095238095238,
+          "precision": 0.7194960212201591,
+          "recall": 0.7359230220955096,
+          "f1": 0.7262905162064826
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.7261904761904762,
+          "precision": 0.684408546477512,
+          "recall": 0.7380952380952381,
+          "f1": 0.6942196418797404
+        },
+        "toxic": {
+          "n_samples": 58,
+          "accuracy": 0.9655172413793104,
+          "precision": 0.75,
+          "recall": 0.9821428571428572,
+          "f1": 0.8242424242424242
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.2787090798576488,
+          "js_divergence": 0.05423684052771057
+        }
+      }
+    },
+    {
+      "fold": 1,
+      "best_params": {
+        "dropout": 0.23085562232445592,
+        "lr": 0.0005227270589225132,
+        "weight_decay": 0.00461701039547356,
+        "backbone_lr_ratio": 0.12087164274488635,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 11,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.3552515818625956,
+          "rmse": 0.5960298498083763,
+          "mae": 0.4518711648970133,
+          "r2": -0.08481203101542345
+        },
+        "delivery": {
+          "n_samples": 61,
+          "mse": 1.078371115810122,
+          "rmse": 1.038446491548853,
+          "mae": 0.6491757012354057,
+          "r2": 0.18375080395351728
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6547619047619048,
+          "precision": 0.5583333333333333,
+          "recall": 0.5634920634920635,
+          "f1": 0.5598915989159892
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.5714285714285714,
+          "precision": 0.5153903903903904,
+          "recall": 0.5209076637648066,
+          "f1": 0.507505844715147
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9508196721311475,
+          "precision": 0.75,
+          "recall": 0.9741379310344828,
+          "f1": 0.8200589970501475
+        },
+        "biodist": {
+          "n_samples": 61,
+          "kl_divergence": 0.5193434798072222,
+          "js_divergence": 0.12637397289897973
+        }
+      }
+    },
+    {
+      "fold": 2,
+      "best_params": {
+        "dropout": 0.318342405407684,
+        "lr": 0.0008937362022799393,
+        "weight_decay": 0.06633867694372529,
+        "backbone_lr_ratio": 0.20906844321970927,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 10,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.3646369784573562,
+          "rmse": 0.6038517851736105,
+          "mae": 0.47954003302203047,
+          "r2": 0.2798484825035997
+        },
+        "delivery": {
+          "n_samples": 60,
+          "mse": 0.6872133280793995,
+          "rmse": 0.828983309891942,
+          "mae": 0.5440382711666947,
+          "r2": 0.08519994983143697
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7380952380952381,
+          "precision": 0.6785714285714286,
+          "recall": 0.7052785923753666,
+          "f1": 0.6867796610169492
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6547619047619048,
+          "precision": 0.6053846153846153,
+          "recall": 0.6720354808590102,
+          "f1": 0.6203502415458937
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9344262295081968,
+          "precision": 0.7142857142857143,
+          "recall": 0.9655172413793103,
+          "f1": 0.7821428571428571
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.35680852482645714,
+          "js_divergence": 0.09025645010360116
+        }
+      }
+    },
+    {
+      "fold": 3,
+      "best_params": {
+        "dropout": 0.12612237693012526,
+        "lr": 0.0009464809649969671,
+        "weight_decay": 0.07051438027639334,
+        "backbone_lr_ratio": 0.9343703462702573,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 19,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.6485083926359867,
+          "rmse": 1.2839425192102591,
+          "mae": 0.5286768473437932,
+          "r2": 0.06759108835091887
+        },
+        "delivery": {
+          "n_samples": 59,
+          "mse": 0.6919949158910226,
+          "rmse": 0.8318623178693831,
+          "mae": 0.5334780438945202,
+          "r2": 0.3272987252110757
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7142857142857143,
+          "precision": 0.6375,
+          "recall": 0.6451612903225806,
+          "f1": 0.6407697790449038
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.5952380952380952,
+          "precision": 0.5380952380952381,
+          "recall": 0.5259426847662142,
+          "f1": 0.5062561094819159
+        },
+        "toxic": {
+          "n_samples": 60,
+          "accuracy": 0.9333333333333333,
+          "precision": 0.7142857142857143,
+          "recall": 0.9649122807017544,
+          "f1": 0.7818181818181817
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.2642989361691452,
+          "js_divergence": 0.06629497269911663
+        }
+      }
+    },
+    {
+      "fold": 4,
+      "best_params": {
+        "dropout": 0.20496607857127885,
+        "lr": 0.0009573095972231165,
+        "weight_decay": 0.00038069869878401156,
+        "backbone_lr_ratio": 0.04121720375781922,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 17,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.7197053265910517,
+          "rmse": 0.8483544816826583,
+          "mae": 0.511052241144214,
+          "r2": 0.036944874168868824
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 1.0001767893259463,
+          "rmse": 1.0000883907565103,
+          "mae": 0.60194506123662,
+          "r2": 0.03284555574221559
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7261904761904762,
+          "precision": 0.6836007130124777,
+          "recall": 0.7265395894428153,
+          "f1": 0.6891391794046662
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6904761904761905,
+          "precision": 0.6314814814814814,
+          "recall": 0.6444444444444445,
+          "f1": 0.6326984475152934
+        },
+        "toxic": {
+          "n_samples": 59,
+          "accuracy": 0.9661016949152542,
+          "precision": 0.8,
+          "recall": 0.9821428571428572,
+          "f1": 0.865909090909091
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.2786833611146747,
+          "js_divergence": 0.07839598379605058
+        }
+      }
+    },
+    {
+      "fold": 0,
+      "best_params": {
+        "dropout": 0.3980773018180746,
+        "lr": 0.0009896030589860336,
+        "weight_decay": 0.0002988554096805342,
+        "backbone_lr_ratio": 0.2882797738700643,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 14,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.5048982722984092,
+          "rmse": 1.2267429528219875,
+          "mae": 0.5805474416723093,
+          "r2": 0.07896521442969773
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 0.6736236117326778,
+          "rmse": 0.8207457655892462,
+          "mae": 0.6456835756142592,
+          "r2": 0.1437837404002562
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7857142857142857,
+          "precision": 0.7305773342836779,
+          "recall": 0.7305773342836779,
+          "f1": 0.7305773342836779
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6785714285714286,
+          "precision": 0.6646103896103895,
+          "recall": 0.7014285714285715,
+          "f1": 0.6618818596621038
+        },
+        "toxic": {
+          "n_samples": 58,
+          "accuracy": 0.9827586206896551,
+          "precision": 0.9912280701754386,
+          "recall": 0.75,
+          "f1": 0.8289085545722714
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.2890762442571175,
+          "js_divergence": 0.05687567953491692
+        }
+      }
+    },
+    {
+      "fold": 1,
+      "best_params": {
+        "dropout": 0.23085562232445592,
+        "lr": 0.0005227270589225132,
+        "weight_decay": 0.00461701039547356,
+        "backbone_lr_ratio": 0.12087164274488635,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 11,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.34313442749289724,
+          "rmse": 0.5857767727495665,
+          "mae": 0.44384193185362075,
+          "r2": -0.04781054949350838
+        },
+        "delivery": {
+          "n_samples": 61,
+          "mse": 1.099845523641604,
+          "rmse": 1.0487352018701404,
+          "mae": 0.6761384357499783,
+          "r2": 0.1674962252922061
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6547619047619048,
+          "precision": 0.6215277777777778,
+          "recall": 0.6587301587301587,
+          "f1": 0.6149834044570887
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.5595238095238095,
+          "precision": 0.5017255158384191,
+          "recall": 0.5057561486132914,
+          "f1": 0.4936173861292259
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9508196721311475,
+          "precision": 0.75,
+          "recall": 0.9741379310344828,
+          "f1": 0.8200589970501475
+        },
+        "biodist": {
+          "n_samples": 61,
+          "kl_divergence": 0.5199198163542676,
+          "js_divergence": 0.12515294319531267
+        }
+      }
+    },
+    {
+      "fold": 2,
+      "best_params": {
+        "dropout": 0.318342405407684,
+        "lr": 0.0008937362022799393,
+        "weight_decay": 0.06633867694372529,
+        "backbone_lr_ratio": 0.20906844321970927,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 10,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.4877212975355261,
+          "rmse": 0.6983704586646876,
+          "mae": 0.5448984429905457,
+          "r2": 0.036759151467687046
+        },
+        "delivery": {
+          "n_samples": 60,
+          "mse": 0.7067592342133922,
+          "rmse": 0.8406897371880974,
+          "mae": 0.5448016864635671,
+          "r2": 0.05918096099439174
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7380952380952381,
+          "precision": 0.6851851851851851,
+          "recall": 0.7199413489736071,
+          "f1": 0.6936339522546419
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6785714285714286,
+          "precision": 0.6240783117213323,
+          "recall": 0.6851073762838468,
+          "f1": 0.6414141414141414
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9344262295081968,
+          "precision": 0.7142857142857143,
+          "recall": 0.9655172413793103,
+          "f1": 0.7821428571428571
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.38840109926844824,
+          "js_divergence": 0.09646626485112685
+        }
+      }
+    },
+    {
+      "fold": 3,
+      "best_params": {
+        "dropout": 0.12612237693012526,
+        "lr": 0.0009464809649969671,
+        "weight_decay": 0.07051438027639334,
+        "backbone_lr_ratio": 0.9343703462702573,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 19,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.9244709959119324,
+          "rmse": 1.3872530396117113,
+          "mae": 0.5739306664004563,
+          "r2": -0.08849546342266934
+        },
+        "delivery": {
+          "n_samples": 59,
+          "mse": 0.7217060909432494,
+          "rmse": 0.849532866311392,
+          "mae": 0.5202912685256136,
+          "r2": 0.2984159330486872
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7023809523809523,
+          "precision": 0.6267796610169492,
+          "recall": 0.6370967741935484,
+          "f1": 0.6307367680675224
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6190476190476191,
+          "precision": 0.5577586206896551,
+          "recall": 0.5861990950226245,
+          "f1": 0.5614861329147044
+        },
+        "toxic": {
+          "n_samples": 60,
+          "accuracy": 0.9333333333333333,
+          "precision": 0.7142857142857143,
+          "recall": 0.9649122807017544,
+          "f1": 0.7818181818181817
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.2617822353923735,
+          "js_divergence": 0.06083979574849321
+        }
+      }
+    },
+    {
+      "fold": 4,
+      "best_params": {
+        "dropout": 0.20496607857127885,
+        "lr": 0.0009573095972231165,
+        "weight_decay": 0.00038069869878401156,
+        "backbone_lr_ratio": 0.04121720375781922,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 17,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.7148724493029326,
+          "rmse": 0.8455013005920999,
+          "mae": 0.5005925307438398,
+          "r2": 0.04341186430062394
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 1.0380505423911823,
+          "rmse": 1.018847654161888,
+          "mae": 0.6400383858588236,
+          "r2": -0.0037777382480910227
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7738095238095238,
+          "precision": 0.7213166144200627,
+          "recall": 0.7587976539589443,
+          "f1": 0.7325289089994973
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6904761904761905,
+          "precision": 0.6308992939427722,
+          "recall": 0.6444444444444445,
+          "f1": 0.633663610371685
+        },
+        "toxic": {
+          "n_samples": 59,
+          "accuracy": 0.9661016949152542,
+          "precision": 0.8,
+          "recall": 0.9821428571428572,
+          "f1": 0.865909090909091
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.2701739757789761,
+          "js_divergence": 0.07372463442305974
+        }
+      }
+    },
+    {
+      "fold": 0,
+      "best_params": {
+        "dropout": 0.3980773018180746,
+        "lr": 0.0009896030589860336,
+        "weight_decay": 0.0002988554096805342,
+        "backbone_lr_ratio": 0.2882797738700643,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 14,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.4541105583695868,
+          "rmse": 1.2058650664023678,
+          "mae": 0.5784477213150766,
+          "r2": 0.11004854548874443
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 0.59667434347555,
+          "rmse": 0.7724469842491134,
+          "mae": 0.6142617367079546,
+          "r2": 0.24159090377534476
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.75,
+          "precision": 0.6916445623342176,
+          "recall": 0.7059871703492516,
+          "f1": 0.6974789915966386
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.7142857142857143,
+          "precision": 0.6695636813283872,
+          "recall": 0.7142857142857144,
+          "f1": 0.6750809541465826
+        },
+        "toxic": {
+          "n_samples": 58,
+          "accuracy": 0.9655172413793104,
+          "precision": 0.75,
+          "recall": 0.9821428571428572,
+          "f1": 0.8242424242424242
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.2587242230390428,
+          "js_divergence": 0.052354750762398025
+        }
+      }
+    },
+    {
+      "fold": 1,
+      "best_params": {
+        "dropout": 0.23085562232445592,
+        "lr": 0.0005227270589225132,
+        "weight_decay": 0.00461701039547356,
+        "backbone_lr_ratio": 0.12087164274488635,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 11,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.36298737292148275,
+          "rmse": 0.6024843341710079,
+          "mae": 0.45925771394034937,
+          "r2": -0.1084343866601285
+        },
+        "delivery": {
+          "n_samples": 61,
+          "mse": 1.0462031939942293,
+          "rmse": 1.022840747132333,
+          "mae": 0.634948962077987,
+          "r2": 0.20809960181702725
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7023809523809523,
+          "precision": 0.6514423076923077,
+          "recall": 0.6904761904761905,
+          "f1": 0.6554552912223134
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.5476190476190477,
+          "precision": 0.48089218250508575,
+          "recall": 0.48011512297226583,
+          "f1": 0.47062888038209943
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9508196721311475,
+          "precision": 0.75,
+          "recall": 0.9741379310344828,
+          "f1": 0.8200589970501475
+        },
+        "biodist": {
+          "n_samples": 61,
+          "kl_divergence": 0.5244687341232338,
+          "js_divergence": 0.12337098779147845
+        }
+      }
+    },
+    {
+      "fold": 2,
+      "best_params": {
+        "dropout": 0.318342405407684,
+        "lr": 0.0008937362022799393,
+        "weight_decay": 0.06633867694372529,
+        "backbone_lr_ratio": 0.20906844321970927,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 10,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.4579348623566749,
+          "rmse": 0.6767088460753818,
+          "mae": 0.5252955685635763,
+          "r2": 0.09558682875266244
+        },
+        "delivery": {
+          "n_samples": 60,
+          "mse": 0.6918557253424359,
+          "rmse": 0.8317786516510483,
+          "mae": 0.5500254434921469,
+          "r2": 0.07902011443593149
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6785714285714286,
+          "precision": 0.6428571428571428,
+          "recall": 0.6796187683284457,
+          "f1": 0.6415362731152205
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6785714285714286,
+          "precision": 0.6222222222222222,
+          "recall": 0.6851073762838468,
+          "f1": 0.636502849002849
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9344262295081968,
+          "precision": 0.7142857142857143,
+          "recall": 0.9655172413793103,
+          "f1": 0.7821428571428571
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.3340915841461506,
+          "js_divergence": 0.08509576023385822
+        }
+      }
+    },
+    {
+      "fold": 3,
+      "best_params": {
+        "dropout": 0.12612237693012526,
+        "lr": 0.0009464809649969671,
+        "weight_decay": 0.07051438027639334,
+        "backbone_lr_ratio": 0.9343703462702573,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 19,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.8202166309492056,
+          "rmse": 1.3491540427057267,
+          "mae": 0.5770047424306408,
+          "r2": -0.02952839998289769
+        },
+        "delivery": {
+          "n_samples": 59,
+          "mse": 0.8240920088267009,
+          "rmse": 0.9077951359347002,
+          "mae": 0.5719114731271135,
+          "r2": 0.1988846563023151
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7142857142857143,
+          "precision": 0.6375,
+          "recall": 0.6451612903225806,
+          "f1": 0.6407697790449038
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.5833333333333334,
+          "precision": 0.475235446249939,
+          "recall": 0.4788838612368025,
+          "f1": 0.4725999853769101
+        },
+        "toxic": {
+          "n_samples": 60,
+          "accuracy": 0.9333333333333333,
+          "precision": 0.7142857142857143,
+          "recall": 0.9649122807017544,
+          "f1": 0.7818181818181817
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.26692129352633476,
+          "js_divergence": 0.061695485072966134
+        }
+      }
+    },
+    {
+      "fold": 4,
+      "best_params": {
+        "dropout": 0.20496607857127885,
+        "lr": 0.0009573095972231165,
+        "weight_decay": 0.00038069869878401156,
+        "backbone_lr_ratio": 0.04121720375781922,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 17,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.713518768290232,
+          "rmse": 0.844700401497615,
+          "mae": 0.5241211098341626,
+          "r2": 0.045223257644336656
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 1.0365573602046312,
+          "rmse": 1.018114610544722,
+          "mae": 0.6499280054764501,
+          "r2": -0.0023338557232996937
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.75,
+          "precision": 0.6949843260188088,
+          "recall": 0.7280058651026393,
+          "f1": 0.7043740573152337
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.7023809523809523,
+          "precision": 0.639751552795031,
+          "recall": 0.6722222222222222,
+          "f1": 0.6522525093953665
+        },
+        "toxic": {
+          "n_samples": 59,
+          "accuracy": 0.9661016949152542,
+          "precision": 0.8,
+          "recall": 0.9821428571428572,
+          "f1": 0.865909090909091
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.3119448408030975,
+          "js_divergence": 0.08658354575321366
+        }
+      }
+    }
+  ],
+  "summary_stats": {
+    "size": {
+      "mse_mean": 0.9648704139917805,
+      "mse_std": 0.5891656318820191,
+      "rmse_mean": 0.9346718018959568,
+      "rmse_std": 0.3020911066753635,
+      "mae_mean": 0.5236781804611128,
+      "mae_std": 0.047009951044540414,
+      "r2_mean": 0.030359535294768855,
+      "r2_std": 0.09435084837045356
+    },
+    "delivery": {
+      "mse_mean": 0.8308453271717486,
+      "mse_std": 0.18767747504329296,
+      "rmse_mean": 0.9057064857810794,
+      "rmse_std": 0.10266980464496812,
+      "mae_mean": 0.5977185404312217,
+      "mae_std": 0.049065196777405395,
+      "r2_mean": 0.1530343582905001,
+      "r2_std": 0.10421669155896922
+    },
+    "pdi": {
+      "accuracy_mean": 0.7238095238095238,
+      "accuracy_std": 0.039411774660111545,
+      "precision_mean": 0.665421093848235,
+      "precision_std": 0.0446473083216482,
+      "recall_mean": 0.6887191408298253,
+      "recall_std": 0.04935509276909138,
+      "f1_mean": 0.669663032996382,
+      "f1_std": 0.04712559651378041
+    },
+    "ee": {
+      "accuracy_mean": 0.6460317460317461,
+      "accuracy_std": 0.058558307129195915,
+      "precision_mean": 0.5894331659154981,
+      "precision_std": 0.06893389274036124,
+      "recall_mean": 0.6169983629815563,
+      "recall_std": 0.08823505436531638,
+      "f1_mean": 0.5906772395955772,
+      "f1_std": 0.07683138902833272
+    },
+    "toxic": {
+      "accuracy_mean": 0.9511890595408049,
+      "accuracy_std": 0.016071213375949578,
+      "precision_mean": 0.7617961570593149,
+      "precision_std": 0.0689235340767993,
+      "recall_mean": 0.958294443004062,
+      "recall_std": 0.05613748087593979,
+      "f1_mean": 0.8151453855878633,
+      "f1_std": 0.03136647604932657
+    },
+    "biodist": {
+      "kl_divergence_mean": 0.34155649523094606,
+      "kl_divergence_std": 0.09686930050549591,
+      "js_divergence_mean": 0.08251453782615217,
+      "js_divergence_std": 0.024921020774773404
+    }
+  }
+}
\ No newline at end of file
diff --git a/results/pretrained_encoders/graphmvp_pretrain_s42_summary.json b/results/pretrained_encoders/graphmvp_pretrain_s42_summary.json
new file mode 100644
index 0000000..1614ce7
--- /dev/null
+++ b/results/pretrained_encoders/graphmvp_pretrain_s42_summary.json
@@ -0,0 +1,932 @@
+{
+  "fold_results": [
+    {
+      "fold": 0,
+      "best_params": {
+        "dropout": 0.3980773018180746,
+        "lr": 0.0009896030589860336,
+        "weight_decay": 0.0002988554096805342,
+        "backbone_lr_ratio": 0.2882797738700643,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 15,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.5221819405605161,
+          "rmse": 1.2337673770044806,
+          "mae": 0.5776655403784958,
+          "r2": 0.06838718401748567
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 0.614112021357094,
+          "rmse": 0.7836529980527696,
+          "mae": 0.6216259215926302,
+          "r2": 0.21942656293011076
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7976190476190477,
+          "precision": 0.7458333333333333,
+          "recall": 0.7523164647184604,
+          "f1": 0.7489010022859153
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6785714285714286,
+          "precision": 0.6691176470588235,
+          "recall": 0.7114285714285714,
+          "f1": 0.6657848324514991
+        },
+        "toxic": {
+          "n_samples": 58,
+          "accuracy": 0.9655172413793104,
+          "precision": 0.75,
+          "recall": 0.9821428571428572,
+          "f1": 0.8242424242424242
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.2940147973370504,
+          "js_divergence": 0.06236386129092423
+        }
+      }
+    },
+    {
+      "fold": 1,
+      "best_params": {
+        "dropout": 0.2902385468029013,
+        "lr": 0.00018739040729381568,
+        "weight_decay": 0.08181191083032673,
+        "backbone_lr_ratio": 0.6566372495858538,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 12,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.2934246330317906,
+          "rmse": 0.5416868403716215,
+          "mae": 0.4069202210133274,
+          "r2": 0.10398548983157396
+        },
+        "delivery": {
+          "n_samples": 61,
+          "mse": 1.156314335236939,
+          "rmse": 1.0753205732417375,
+          "mae": 0.6882066883329974,
+          "r2": 0.12475340569084314
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6547619047619048,
+          "precision": 0.5583333333333333,
+          "recall": 0.5634920634920635,
+          "f1": 0.5598915989159892
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.5595238095238095,
+          "precision": 0.48693693693693696,
+          "recall": 0.49312592169735026,
+          "f1": 0.47630082959336323
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9508196721311475,
+          "precision": 0.75,
+          "recall": 0.9741379310344828,
+          "f1": 0.8200589970501475
+        },
+        "biodist": {
+          "n_samples": 61,
+          "kl_divergence": 0.5092761936607958,
+          "js_divergence": 0.13355010688479413
+        }
+      }
+    },
+    {
+      "fold": 2,
+      "best_params": {
+        "dropout": 0.18117503720006686,
+        "lr": 8.135394049241399e-05,
+        "weight_decay": 0.015076469897649124,
+        "backbone_lr_ratio": 0.9586175198423679,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 18,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.34434236619012065,
+          "rmse": 0.586806924115693,
+          "mae": 0.46669395072967745,
+          "r2": 0.3199299791282213
+        },
+        "delivery": {
+          "n_samples": 60,
+          "mse": 0.6575735692519624,
+          "rmse": 0.810909100486585,
+          "mae": 0.55640976827126,
+          "r2": 0.12465560610938209
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6190476190476191,
+          "precision": 0.6190476190476191,
+          "recall": 0.653958944281525,
+          "f1": 0.5961538461538461
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6666666666666666,
+          "precision": 0.6166666666666667,
+          "recall": 0.6785714285714285,
+          "f1": 0.6334814122048165
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9344262295081968,
+          "precision": 0.7142857142857143,
+          "recall": 0.9655172413793103,
+          "f1": 0.7821428571428571
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.41784407477442437,
+          "js_divergence": 0.11469543309672324
+        }
+      }
+    },
+    {
+      "fold": 3,
+      "best_params": {
+        "dropout": 0.1799936812783694,
+        "lr": 0.00033865324369584493,
+        "weight_decay": 0.002833833442807743,
+        "backbone_lr_ratio": 0.3278334190266754,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 8,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.6764751937886317,
+          "rmse": 1.2947877022078298,
+          "mae": 0.622077302143247,
+          "r2": 0.05177285245868457
+        },
+        "delivery": {
+          "n_samples": 59,
+          "mse": 0.725000410122951,
+          "rmse": 0.8514695591287753,
+          "mae": 0.6181600616487154,
+          "r2": 0.2952134634049711
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6428571428571429,
+          "precision": 0.5714285714285714,
+          "recall": 0.5821114369501467,
+          "f1": 0.5728813559322035
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6666666666666666,
+          "precision": 0.5873015873015873,
+          "recall": 0.6022121669180492,
+          "f1": 0.5872162485065711
+        },
+        "toxic": {
+          "n_samples": 60,
+          "accuracy": 0.9333333333333333,
+          "precision": 0.7142857142857143,
+          "recall": 0.9649122807017544,
+          "f1": 0.7818181818181817
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.4920884182489342,
+          "js_divergence": 0.12950821301606347
+        }
+      }
+    },
+    {
+      "fold": 4,
+      "best_params": {
+        "dropout": 0.19398015625011103,
+        "lr": 0.0008450080181209037,
+        "weight_decay": 0.002901987198512681,
+        "backbone_lr_ratio": 0.5310897731873246,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 16,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.7301598129307739,
+          "rmse": 0.8544938928575054,
+          "mae": 0.4922567930327551,
+          "r2": 0.022955472832782875
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 1.125780851035229,
+          "rmse": 1.0610282046370063,
+          "mae": 0.6240321383650961,
+          "r2": -0.08861149844600491
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.75,
+          "precision": 0.683728813559322,
+          "recall": 0.6986803519061584,
+          "f1": 0.6898188851767189
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6666666666666666,
+          "precision": 0.6060606060606061,
+          "recall": 0.6418803418803419,
+          "f1": 0.6178929765886287
+        },
+        "toxic": {
+          "n_samples": 59,
+          "accuracy": 0.9661016949152542,
+          "precision": 0.8,
+          "recall": 0.9821428571428572,
+          "f1": 0.865909090909091
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.26640631927709985,
+          "js_divergence": 0.07248344781159008
+        }
+      }
+    },
+    {
+      "fold": 0,
+      "best_params": {
+        "dropout": 0.3980773018180746,
+        "lr": 0.0009896030589860336,
+        "weight_decay": 0.0002988554096805342,
+        "backbone_lr_ratio": 0.2882797738700643,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 15,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.5188104198487684,
+          "rmse": 1.2324002677088188,
+          "mae": 0.5873989220259779,
+          "r2": 0.07045063768272763
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 0.6666016049923045,
+          "rmse": 0.8164567379796094,
+          "mae": 0.6511167181206161,
+          "r2": 0.15270913470266467
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7857142857142857,
+          "precision": 0.7305773342836779,
+          "recall": 0.7305773342836779,
+          "f1": 0.7305773342836779
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.7023809523809523,
+          "precision": 0.691027926322044,
+          "recall": 0.741904761904762,
+          "f1": 0.6917852553884204
+        },
+        "toxic": {
+          "n_samples": 58,
+          "accuracy": 0.9827586206896551,
+          "precision": 0.9912280701754386,
+          "recall": 0.75,
+          "f1": 0.8289085545722714
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.2601734547154161,
+          "js_divergence": 0.056514013464432016
+        }
+      }
+    },
+    {
+      "fold": 1,
+      "best_params": {
+        "dropout": 0.2902385468029013,
+        "lr": 0.00018739040729381568,
+        "weight_decay": 0.08181191083032673,
+        "backbone_lr_ratio": 0.6566372495858538,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 12,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.3231063887830616,
+          "rmse": 0.5684244794016717,
+          "mae": 0.42929340717160985,
+          "r2": 0.01334796030441776
+        },
+        "delivery": {
+          "n_samples": 61,
+          "mse": 1.1342861224694418,
+          "rmse": 1.0650286956084525,
+          "mae": 0.6788244451106084,
+          "r2": 0.14142717476551148
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.5595238095238095,
+          "precision": 0.5477272727272727,
+          "recall": 0.5634920634920635,
+          "f1": 0.5238241152137275
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.5476190476190477,
+          "precision": 0.473011973011973,
+          "recall": 0.4779744065458351,
+          "f1": 0.46420797118471535
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9508196721311475,
+          "precision": 0.75,
+          "recall": 0.9741379310344828,
+          "f1": 0.8200589970501475
+        },
+        "biodist": {
+          "n_samples": 61,
+          "kl_divergence": 0.5813128628991394,
+          "js_divergence": 0.14714358123767265
+        }
+      }
+    },
+    {
+      "fold": 2,
+      "best_params": {
+        "dropout": 0.18117503720006686,
+        "lr": 8.135394049241399e-05,
+        "weight_decay": 0.015076469897649124,
+        "backbone_lr_ratio": 0.9586175198423679,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 18,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.43267246099702694,
+          "rmse": 0.6577784284977936,
+          "mae": 0.5142740875349513,
+          "r2": 0.14547962007547455
+        },
+        "delivery": {
+          "n_samples": 60,
+          "mse": 0.6198350807939489,
+          "rmse": 0.7872960566355892,
+          "mae": 0.5207243816383804,
+          "r2": 0.1748920752290377
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6904761904761905,
+          "precision": 0.6418269230769231,
+          "recall": 0.6730205278592376,
+          "f1": 0.6452241715399609
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6785714285714286,
+          "precision": 0.6190476190476191,
+          "recall": 0.6757703081232492,
+          "f1": 0.63584229390681
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9344262295081968,
+          "precision": 0.7142857142857143,
+          "recall": 0.9655172413793103,
+          "f1": 0.7821428571428571
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.42003529077921203,
+          "js_divergence": 0.11409932566058836
+        }
+      }
+    },
+    {
+      "fold": 3,
+      "best_params": {
+        "dropout": 0.1799936812783694,
+        "lr": 0.00033865324369584493,
+        "weight_decay": 0.002833833442807743,
+        "backbone_lr_ratio": 0.3278334190266754,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 8,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.709633019803758,
+          "rmse": 1.3075293571479603,
+          "mae": 0.629169082886214,
+          "r2": 0.03301853333873328
+        },
+        "delivery": {
+          "n_samples": 59,
+          "mse": 0.7678785210635485,
+          "rmse": 0.8762867801487983,
+          "mae": 0.6255631820472368,
+          "r2": 0.2535308451835053
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6666666666666666,
+          "precision": 0.5888594164456233,
+          "recall": 0.5982404692082112,
+          "f1": 0.5916666666666667
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6547619047619048,
+          "precision": 0.5874977416440831,
+          "recall": 0.6237556561085973,
+          "f1": 0.5958755389330425
+        },
+        "toxic": {
+          "n_samples": 60,
+          "accuracy": 0.9333333333333333,
+          "precision": 0.7142857142857143,
+          "recall": 0.9649122807017544,
+          "f1": 0.7818181818181817
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.46770485035756304,
+          "js_divergence": 0.12378056507326861
+        }
+      }
+    },
+    {
+      "fold": 4,
+      "best_params": {
+        "dropout": 0.19398015625011103,
+        "lr": 0.0008450080181209037,
+        "weight_decay": 0.002901987198512681,
+        "backbone_lr_ratio": 0.5310897731873246,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 16,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.6133402094103233,
+          "rmse": 0.7831603982648275,
+          "mae": 0.4741146452669498,
+          "r2": 0.17927461319654014
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 1.0688240926391441,
+          "rmse": 1.0338394907523818,
+          "mae": 0.6684364553155571,
+          "r2": -0.0335352533249651
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7857142857142857,
+          "precision": 0.721875,
+          "recall": 0.7082111436950147,
+          "f1": 0.7142857142857142
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6428571428571429,
+          "precision": 0.5752393980848153,
+          "recall": 0.6188034188034188,
+          "f1": 0.5883517571357862
+        },
+        "toxic": {
+          "n_samples": 59,
+          "accuracy": 0.9661016949152542,
+          "precision": 0.8,
+          "recall": 0.9821428571428572,
+          "f1": 0.865909090909091
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.28068032170399054,
+          "js_divergence": 0.07478332635258625
+        }
+      }
+    },
+    {
+      "fold": 0,
+      "best_params": {
+        "dropout": 0.3980773018180746,
+        "lr": 0.0009896030589860336,
+        "weight_decay": 0.0002988554096805342,
+        "backbone_lr_ratio": 0.2882797738700643,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 15,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.3818212591739194,
+          "rmse": 1.1755089362373727,
+          "mae": 0.5521946805697607,
+          "r2": 0.1542913759905169
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 0.6972128277816507,
+          "rmse": 0.8349927112146852,
+          "mae": 0.6792561926333041,
+          "r2": 0.11380042333630946
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7857142857142857,
+          "precision": 0.7305773342836779,
+          "recall": 0.7305773342836779,
+          "f1": 0.7305773342836779
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.7261904761904762,
+          "precision": 0.6891715244656421,
+          "recall": 0.7380952380952381,
+          "f1": 0.6965863304906686
+        },
+        "toxic": {
+          "n_samples": 58,
+          "accuracy": 0.9655172413793104,
+          "precision": 0.75,
+          "recall": 0.9821428571428572,
+          "f1": 0.8242424242424242
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.30701952754304446,
+          "js_divergence": 0.057564049906523536
+        }
+      }
+    },
+    {
+      "fold": 1,
+      "best_params": {
+        "dropout": 0.2902385468029013,
+        "lr": 0.00018739040729381568,
+        "weight_decay": 0.08181191083032673,
+        "backbone_lr_ratio": 0.6566372495858538,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 12,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.34204974602902377,
+          "rmse": 0.5848501910994163,
+          "mae": 0.44300510672231513,
+          "r2": -0.04449831793169534
+        },
+        "delivery": {
+          "n_samples": 61,
+          "mse": 1.1297399798287129,
+          "rmse": 1.0628922710363045,
+          "mae": 0.6765872548166357,
+          "r2": 0.14486827701797667
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6428571428571429,
+          "precision": 0.5374198146828225,
+          "recall": 0.5396825396825397,
+          "f1": 0.5381231671554252
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.5714285714285714,
+          "precision": 0.5202020202020202,
+          "recall": 0.5292564578278864,
+          "f1": 0.5119174942704355
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9344262295081968,
+          "precision": 0.7142857142857143,
+          "recall": 0.9655172413793103,
+          "f1": 0.7821428571428571
+        },
+        "biodist": {
+          "n_samples": 61,
+          "kl_divergence": 0.5145521130605141,
+          "js_divergence": 0.1322571633039113
+        }
+      }
+    },
+    {
+      "fold": 2,
+      "best_params": {
+        "dropout": 0.18117503720006686,
+        "lr": 8.135394049241399e-05,
+        "weight_decay": 0.015076469897649124,
+        "backbone_lr_ratio": 0.9586175198423679,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 18,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.4742411460721128,
+          "rmse": 0.6886516870465887,
+          "mae": 0.5345059656759813,
+          "r2": 0.06338220975850639
+        },
+        "delivery": {
+          "n_samples": 60,
+          "mse": 0.6525032683185743,
+          "rmse": 0.807776744106052,
+          "mae": 0.5494959851183618,
+          "r2": 0.13140505545605896
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.5952380952380952,
+          "precision": 0.6319444444444444,
+          "recall": 0.6671554252199414,
+          "f1": 0.583673469387755
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6666666666666666,
+          "precision": 0.6091142490372272,
+          "recall": 0.669234360410831,
+          "f1": 0.6249493532803491
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9344262295081968,
+          "precision": 0.7142857142857143,
+          "recall": 0.9655172413793103,
+          "f1": 0.7821428571428571
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.4235028564925827,
+          "js_divergence": 0.11443098458109038
+        }
+      }
+    },
+    {
+      "fold": 3,
+      "best_params": {
+        "dropout": 0.1799936812783694,
+        "lr": 0.00033865324369584493,
+        "weight_decay": 0.002833833442807743,
+        "backbone_lr_ratio": 0.3278334190266754,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 8,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.7043255314620636,
+          "rmse": 1.305498192822213,
+          "mae": 0.6349209440782307,
+          "r2": 0.036020489197966854
+        },
+        "delivery": {
+          "n_samples": 59,
+          "mse": 0.7865588440055382,
+          "rmse": 0.886881527604188,
+          "mae": 0.6117917100851566,
+          "r2": 0.23537135185779967
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6904761904761905,
+          "precision": 0.616710875331565,
+          "recall": 0.6290322580645161,
+          "f1": 0.6208333333333333
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6785714285714286,
+          "precision": 0.6092796092796093,
+          "recall": 0.636827551533434,
+          "f1": 0.6157297695238403
+        },
+        "toxic": {
+          "n_samples": 60,
+          "accuracy": 0.9333333333333333,
+          "precision": 0.7142857142857143,
+          "recall": 0.9649122807017544,
+          "f1": 0.7818181818181817
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.475069604298859,
+          "js_divergence": 0.12514035771620471
+        }
+      }
+    },
+    {
+      "fold": 4,
+      "best_params": {
+        "dropout": 0.19398015625011103,
+        "lr": 0.0008450080181209037,
+        "weight_decay": 0.002901987198512681,
+        "backbone_lr_ratio": 0.5310897731873246,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 16,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.8580511263094861,
+          "rmse": 0.9263104913091971,
+          "mae": 0.5776896519043172,
+          "r2": -0.14817899060384732
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 1.043085209273457,
+          "rmse": 1.0213154308407648,
+          "mae": 0.59220993094917,
+          "r2": -0.008646178010456307
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6904761904761905,
+          "precision": 0.6333333333333333,
+          "recall": 0.6583577712609971,
+          "f1": 0.6379310344827587
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.7261904761904762,
+          "precision": 0.6652777777777779,
+          "recall": 0.7064102564102565,
+          "f1": 0.6809523809523809
+        },
+        "toxic": {
+          "n_samples": 59,
+          "accuracy": 0.9661016949152542,
+          "precision": 0.8,
+          "recall": 0.9821428571428572,
+          "f1": 0.865909090909091
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.24694979259516875,
+          "js_divergence": 0.06657100087180373
+        }
+      }
+    }
+  ],
+  "summary_stats": {
+    "size": {
+      "mse_mean": 0.9283090169594252,
+      "mse_std": 0.5614727660610882,
+      "rmse_mean": 0.9161103444061993,
+      "rmse_std": 0.29841389684694647,
+      "mae_mean": 0.529478686742254,
+      "mae_std": 0.07272420920077967,
+      "r2_mean": 0.07130794061853929,
+      "r2_std": 0.10242823171449583
+    },
+    "delivery": {
+      "mse_mean": 0.8563537825446997,
+      "mse_std": 0.21315743069248957,
+      "rmse_mean": 0.91834312543158,
+      "rmse_std": 0.1140161677888585,
+      "mae_mean": 0.624162722269715,
+      "mae_std": 0.05009549555609884,
+      "r2_mean": 0.13208402972684966,
+      "r2_std": 0.10281086532328124
+    },
+    "pdi": {
+      "accuracy_mean": 0.6904761904761904,
+      "accuracy_std": 0.07299861769883757,
+      "precision_mean": 0.6372815612874345,
+      "precision_std": 0.06842608292617637,
+      "recall_mean": 0.6499270752265488,
+      "recall_std": 0.06582138762012522,
+      "f1_mean": 0.6322908686064913,
+      "f1_std": 0.07211332053746232
+    },
+    "ee": {
+      "accuracy_mean": 0.6555555555555556,
+      "accuracy_std": 0.05314498108820064,
+      "precision_mean": 0.6003302188598288,
+      "precision_std": 0.06446111150537735,
+      "recall_mean": 0.6363500564172834,
+      "recall_std": 0.07965961750314356,
+      "f1_mean": 0.6057916296274218,
+      "f1_std": 0.07002027044312158
+    },
+    "toxic": {
+      "accuracy_mean": 0.9500961633659415,
+      "accuracy_std": 0.01660762512329349,
+      "precision_mean": 0.7594152046783624,
+      "precision_std": 0.0698998626999633,
+      "recall_mean": 0.9577197303603838,
+      "recall_std": 0.05601633670665784,
+      "f1_mean": 0.8126176429273773,
+      "f1_std": 0.03238005389957549
+    },
+    "biodist": {
+      "kl_divergence_mean": 0.39710869851625297,
+      "kl_divergence_std": 0.10730729317255493,
+      "js_divergence_mean": 0.10165902868454511,
+      "js_divergence_std": 0.03125967849638632
+    }
+  }
+}
\ No newline at end of file
diff --git a/results/pretrained_encoders/grover_pretrain_s42_summary.json b/results/pretrained_encoders/grover_pretrain_s42_summary.json
new file mode 100644
index 0000000..71f08d1
--- /dev/null
+++ b/results/pretrained_encoders/grover_pretrain_s42_summary.json
@@ -0,0 +1,932 @@
+{
+  "fold_results": [
+    {
+      "fold": 0,
+      "best_params": {
+        "dropout": 0.25888363054578495,
+        "lr": 0.0006536264082965735,
+        "weight_decay": 0.00932351457657158,
+        "backbone_lr_ratio": 0.1667894332719967,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 13,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.5789235054424788,
+          "rmse": 1.2565522294924627,
+          "mae": 0.5897082271031946,
+          "r2": 0.03365994962165875
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 0.5843462843532253,
+          "rmse": 0.7644254602989263,
+          "mae": 0.6106570679694414,
+          "r2": 0.2572606108431995
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7261904761904762,
+          "precision": 0.7065217391304348,
+          "recall": 0.7573057733428368,
+          "f1": 0.7039987743220468
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6785714285714286,
+          "precision": 0.6388888888888888,
+          "recall": 0.6842857142857143,
+          "f1": 0.6433139534883722
+        },
+        "toxic": {
+          "n_samples": 58,
+          "accuracy": 0.9655172413793104,
+          "precision": 0.75,
+          "recall": 0.9821428571428572,
+          "f1": 0.8242424242424242
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.295739533929628,
+          "js_divergence": 0.06360146142338208
+        }
+      }
+    },
+    {
+      "fold": 1,
+      "best_params": {
+        "dropout": 0.37015922545878527,
+        "lr": 0.0009352630241255291,
+        "weight_decay": 0.005516105285933501,
+        "backbone_lr_ratio": 0.12042458689005697,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 9,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.3504606454005027,
+          "rmse": 0.5919971667166176,
+          "mae": 0.4625389823867452,
+          "r2": -0.07018221434674188
+        },
+        "delivery": {
+          "n_samples": 61,
+          "mse": 1.0871146054096508,
+          "rmse": 1.0426478817940652,
+          "mae": 0.6687804595490948,
+          "r2": 0.17713261263549906
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6904761904761905,
+          "precision": 0.6067796610169491,
+          "recall": 0.6190476190476191,
+          "f1": 0.6108339272986457
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.5357142857142857,
+          "precision": 0.4496353496353496,
+          "recall": 0.4523333809048095,
+          "f1": 0.4379382564939481
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9508196721311475,
+          "precision": 0.75,
+          "recall": 0.9741379310344828,
+          "f1": 0.8200589970501475
+        },
+        "biodist": {
+          "n_samples": 61,
+          "kl_divergence": 0.382227541927671,
+          "js_divergence": 0.09520406464421978
+        }
+      }
+    },
+    {
+      "fold": 2,
+      "best_params": {
+        "dropout": 0.4042053577703755,
+        "lr": 0.0004438352448528134,
+        "weight_decay": 0.09116356780872863,
+        "backbone_lr_ratio": 0.14631991134515426,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 17,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.3317227949570641,
+          "rmse": 0.5759538132151432,
+          "mae": 0.44960591762459706,
+          "r2": 0.3448534068400446
+        },
+        "delivery": {
+          "n_samples": 60,
+          "mse": 0.6524805809132939,
+          "rmse": 0.8077627008678315,
+          "mae": 0.5519438893922294,
+          "r2": 0.13143525632475672
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7023809523809523,
+          "precision": 0.6586452762923352,
+          "recall": 0.6957478005865103,
+          "f1": 0.662107803700724
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6904761904761905,
+          "precision": 0.640510613250929,
+          "recall": 0.7009803921568626,
+          "f1": 0.6591013004136229
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9344262295081968,
+          "precision": 0.7142857142857143,
+          "recall": 0.9655172413793103,
+          "f1": 0.7821428571428571
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.3461542146527445,
+          "js_divergence": 0.09035975387137388
+        }
+      }
+    },
+    {
+      "fold": 3,
+      "best_params": {
+        "dropout": 0.12612237693012526,
+        "lr": 0.0009464809649969671,
+        "weight_decay": 0.07051438027639334,
+        "backbone_lr_ratio": 0.9343703462702573,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 16,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.630017453835127,
+          "rmse": 1.27672136891145,
+          "mae": 0.57301952361118,
+          "r2": 0.07804970427286173
+        },
+        "delivery": {
+          "n_samples": 59,
+          "mse": 0.7426488745509607,
+          "rmse": 0.8617707784271643,
+          "mae": 0.5635692574304797,
+          "r2": 0.27805705915089884
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7261904761904762,
+          "precision": 0.6489665003563792,
+          "recall": 0.6532258064516129,
+          "f1": 0.6509485094850949
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6190476190476191,
+          "precision": 0.5598290598290598,
+          "recall": 0.5861990950226245,
+          "f1": 0.5606349206349206
+        },
+        "toxic": {
+          "n_samples": 60,
+          "accuracy": 0.9333333333333333,
+          "precision": 0.7142857142857143,
+          "recall": 0.9649122807017544,
+          "f1": 0.7818181818181817
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.32329647019984065,
+          "js_divergence": 0.07342759749750183
+        }
+      }
+    },
+    {
+      "fold": 4,
+      "best_params": {
+        "dropout": 0.20496607857127885,
+        "lr": 0.0009573095972231165,
+        "weight_decay": 0.00038069869878401156,
+        "backbone_lr_ratio": 0.04121720375781922,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 10,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.714982360155395,
+          "rmse": 0.845566295541275,
+          "mae": 0.5158508041640744,
+          "r2": 0.04326478992735272
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 0.9337723075176421,
+          "rmse": 0.966318947096476,
+          "mae": 0.615490146495145,
+          "r2": 0.0970575934388902
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7023809523809523,
+          "precision": 0.6423197492163009,
+          "recall": 0.6664222873900294,
+          "f1": 0.6480643539467068
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6428571428571429,
+          "precision": 0.5733695652173912,
+          "recall": 0.6085470085470085,
+          "f1": 0.5861678004535147
+        },
+        "toxic": {
+          "n_samples": 59,
+          "accuracy": 0.9661016949152542,
+          "precision": 0.8,
+          "recall": 0.9821428571428572,
+          "f1": 0.865909090909091
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.3041751519174415,
+          "js_divergence": 0.0828665483142382
+        }
+      }
+    },
+    {
+      "fold": 0,
+      "best_params": {
+        "dropout": 0.25888363054578495,
+        "lr": 0.0006536264082965735,
+        "weight_decay": 0.00932351457657158,
+        "backbone_lr_ratio": 0.1667894332719967,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 13,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.516128831525372,
+          "rmse": 1.2313118335845603,
+          "mae": 0.5887543337801134,
+          "r2": 0.07209183574368039
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 0.6296859698518799,
+          "rmse": 0.793527548262743,
+          "mae": 0.6343367575699913,
+          "r2": 0.199631134600178
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7976190476190477,
+          "precision": 0.7537037037037038,
+          "recall": 0.7929436920883821,
+          "f1": 0.765709598031173
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.7023809523809523,
+          "precision": 0.6626672044094911,
+          "recall": 0.7147619047619048,
+          "f1": 0.6696677261461316
+        },
+        "toxic": {
+          "n_samples": 58,
+          "accuracy": 0.9655172413793104,
+          "precision": 0.75,
+          "recall": 0.9821428571428572,
+          "f1": 0.8242424242424242
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.2648199540119534,
+          "js_divergence": 0.05676284260509181
+        }
+      }
+    },
+    {
+      "fold": 1,
+      "best_params": {
+        "dropout": 0.37015922545878527,
+        "lr": 0.0009352630241255291,
+        "weight_decay": 0.005516105285933501,
+        "backbone_lr_ratio": 0.12042458689005697,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 9,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.3644874868613839,
+          "rmse": 0.603727990788388,
+          "mae": 0.4844940258633523,
+          "r2": -0.11301520130806475
+        },
+        "delivery": {
+          "n_samples": 61,
+          "mse": 1.0796036294761662,
+          "rmse": 1.039039763183376,
+          "mae": 0.6630113559973533,
+          "r2": 0.1828178799589153
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6428571428571429,
+          "precision": 0.5498305084745763,
+          "recall": 0.5555555555555556,
+          "f1": 0.5509622238061297
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.5238095238095238,
+          "precision": 0.4709239314502473,
+          "recall": 0.48534798534798534,
+          "f1": 0.4678875968992248
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9508196721311475,
+          "precision": 0.75,
+          "recall": 0.9741379310344828,
+          "f1": 0.8200589970501475
+        },
+        "biodist": {
+          "n_samples": 61,
+          "kl_divergence": 0.360009988398715,
+          "js_divergence": 0.08968488957961811
+        }
+      }
+    },
+    {
+      "fold": 2,
+      "best_params": {
+        "dropout": 0.4042053577703755,
+        "lr": 0.0004438352448528134,
+        "weight_decay": 0.09116356780872863,
+        "backbone_lr_ratio": 0.14631991134515426,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 17,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.3763557616627376,
+          "rmse": 0.613478411733239,
+          "mae": 0.48970496371787575,
+          "r2": 0.2567040950520848
+        },
+        "delivery": {
+          "n_samples": 60,
+          "mse": 0.703379623118135,
+          "rmse": 0.8386773057130704,
+          "mae": 0.5631674273948496,
+          "r2": 0.06367980912955817
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7142857142857143,
+          "precision": 0.6752941176470588,
+          "recall": 0.718475073313783,
+          "f1": 0.6785714285714286
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6904761904761905,
+          "precision": 0.6362280701754386,
+          "recall": 0.7009803921568626,
+          "f1": 0.6528571942316682
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9344262295081968,
+          "precision": 0.7142857142857143,
+          "recall": 0.9655172413793103,
+          "f1": 0.7821428571428571
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.33457114910094327,
+          "js_divergence": 0.08453543607257773
+        }
+      }
+    },
+    {
+      "fold": 3,
+      "best_params": {
+        "dropout": 0.12612237693012526,
+        "lr": 0.0009464809649969671,
+        "weight_decay": 0.07051438027639334,
+        "backbone_lr_ratio": 0.9343703462702573,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 16,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.9634922624183433,
+          "rmse": 1.4012466815012778,
+          "mae": 0.6159229520596115,
+          "r2": -0.1105661891750771
+        },
+        "delivery": {
+          "n_samples": 59,
+          "mse": 0.8685113223217268,
+          "rmse": 0.9319395486412876,
+          "mae": 0.6189951948916256,
+          "r2": 0.1557038060862732
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7023809523809523,
+          "precision": 0.6503347534996957,
+          "recall": 0.6810850439882699,
+          "f1": 0.6554552912223134
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.5833333333333334,
+          "precision": 0.5217673814165041,
+          "recall": 0.5463298139768727,
+          "f1": 0.5207287433261615
+        },
+        "toxic": {
+          "n_samples": 60,
+          "accuracy": 0.9,
+          "precision": 0.6666666666666666,
+          "recall": 0.9473684210526316,
+          "f1": 0.7222222222222222
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.24978094522784985,
+          "js_divergence": 0.06060532361779249
+        }
+      }
+    },
+    {
+      "fold": 4,
+      "best_params": {
+        "dropout": 0.20496607857127885,
+        "lr": 0.0009573095972231165,
+        "weight_decay": 0.00038069869878401156,
+        "backbone_lr_ratio": 0.04121720375781922,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 10,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.666445125466178,
+          "rmse": 0.8163609039304723,
+          "mae": 0.49950868456203135,
+          "r2": 0.10821363903831627
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 0.9994847526266784,
+          "rmse": 0.9997423431198053,
+          "mae": 0.6419767608288033,
+          "r2": 0.033514743806195346
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7023809523809523,
+          "precision": 0.6503347534996957,
+          "recall": 0.6810850439882699,
+          "f1": 0.6554552912223134
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6547619047619048,
+          "precision": 0.5829995828118482,
+          "recall": 0.614957264957265,
+          "f1": 0.5934111227214677
+        },
+        "toxic": {
+          "n_samples": 59,
+          "accuracy": 0.9661016949152542,
+          "precision": 0.8,
+          "recall": 0.9821428571428572,
+          "f1": 0.865909090909091
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.2719915000176464,
+          "js_divergence": 0.0728753918681956
+        }
+      }
+    },
+    {
+      "fold": 0,
+      "best_params": {
+        "dropout": 0.25888363054578495,
+        "lr": 0.0006536264082965735,
+        "weight_decay": 0.00932351457657158,
+        "backbone_lr_ratio": 0.1667894332719967,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 13,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.515637174855938,
+          "rmse": 1.2311121698918983,
+          "mae": 0.5759322074836636,
+          "r2": 0.07239274172745469
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 0.6078859434204266,
+          "rmse": 0.7796704069159138,
+          "mae": 0.6284612801311344,
+          "r2": 0.22734028369354697
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.75,
+          "precision": 0.7147058823529412,
+          "recall": 0.7601568068424804,
+          "f1": 0.7211948790896159
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6904761904761905,
+          "precision": 0.6494435612082671,
+          "recall": 0.690952380952381,
+          "f1": 0.6540847855642071
+        },
+        "toxic": {
+          "n_samples": 58,
+          "accuracy": 0.9655172413793104,
+          "precision": 0.75,
+          "recall": 0.9821428571428572,
+          "f1": 0.8242424242424242
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.28451853940500493,
+          "js_divergence": 0.06143098168643295
+        }
+      }
+    },
+    {
+      "fold": 1,
+      "best_params": {
+        "dropout": 0.37015922545878527,
+        "lr": 0.0009352630241255291,
+        "weight_decay": 0.005516105285933501,
+        "backbone_lr_ratio": 0.12042458689005697,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 9,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.36607084927706934,
+          "rmse": 0.6050378907779821,
+          "mae": 0.4742527622064309,
+          "r2": -0.11785022720432559
+        },
+        "delivery": {
+          "n_samples": 61,
+          "mse": 1.072597205810797,
+          "rmse": 1.0356626892047416,
+          "mae": 0.6899142471737549,
+          "r2": 0.18812123758800225
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6309523809523809,
+          "precision": 0.5535714285714286,
+          "recall": 0.5634920634920635,
+          "f1": 0.553421368547419
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.5595238095238095,
+          "precision": 0.49943718228031947,
+          "recall": 0.516245659102802,
+          "f1": 0.5015974929297562
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9508196721311475,
+          "precision": 0.75,
+          "recall": 0.9741379310344828,
+          "f1": 0.8200589970501475
+        },
+        "biodist": {
+          "n_samples": 61,
+          "kl_divergence": 0.33867682492719764,
+          "js_divergence": 0.08667274774942817
+        }
+      }
+    },
+    {
+      "fold": 2,
+      "best_params": {
+        "dropout": 0.4042053577703755,
+        "lr": 0.0004438352448528134,
+        "weight_decay": 0.09116356780872863,
+        "backbone_lr_ratio": 0.14631991134515426,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 17,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.5053228762985142,
+          "rmse": 0.7108606588484935,
+          "mae": 0.544970038779346,
+          "r2": 0.001996347897613271
+        },
+        "delivery": {
+          "n_samples": 60,
+          "mse": 0.7184112972452166,
+          "rmse": 0.8475914683650471,
+          "mae": 0.5859764796448872,
+          "r2": 0.043670045518013456
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7261904761904762,
+          "precision": 0.6836007130124777,
+          "recall": 0.7265395894428153,
+          "f1": 0.6891391794046662
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6666666666666666,
+          "precision": 0.6104363472784525,
+          "recall": 0.6666666666666666,
+          "f1": 0.6224764228882279
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9344262295081968,
+          "precision": 0.7142857142857143,
+          "recall": 0.9655172413793103,
+          "f1": 0.7821428571428571
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.34415499439775205,
+          "js_divergence": 0.08771435148935566
+        }
+      }
+    },
+    {
+      "fold": 3,
+      "best_params": {
+        "dropout": 0.12612237693012526,
+        "lr": 0.0009464809649969671,
+        "weight_decay": 0.07051438027639334,
+        "backbone_lr_ratio": 0.9343703462702573,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 16,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.7695798467912622,
+          "rmse": 1.3302555569480858,
+          "mae": 0.6023058171933854,
+          "r2": -0.0008878489144126256
+        },
+        "delivery": {
+          "n_samples": 59,
+          "mse": 0.7584981089453127,
+          "rmse": 0.8709179691252861,
+          "mae": 0.581826904741258,
+          "r2": 0.2626497202576923
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7261904761904762,
+          "precision": 0.6489665003563792,
+          "recall": 0.6532258064516129,
+          "f1": 0.6509485094850949
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6666666666666666,
+          "precision": 0.571467539552646,
+          "recall": 0.583107088989442,
+          "f1": 0.5753294324722896
+        },
+        "toxic": {
+          "n_samples": 60,
+          "accuracy": 0.9333333333333333,
+          "precision": 0.7142857142857143,
+          "recall": 0.9649122807017544,
+          "f1": 0.7818181818181817
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.26330683934899884,
+          "js_divergence": 0.05955508350882244
+        }
+      }
+    },
+    {
+      "fold": 4,
+      "best_params": {
+        "dropout": 0.20496607857127885,
+        "lr": 0.0009573095972231165,
+        "weight_decay": 0.00038069869878401156,
+        "backbone_lr_ratio": 0.04121720375781922,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 10,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.6649265530357125,
+          "rmse": 0.8154302870483242,
+          "mae": 0.5136514454137623,
+          "r2": 0.1102456775811359
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 0.9676062147836739,
+          "rmse": 0.9836697691723956,
+          "mae": 0.6175258674755179,
+          "r2": 0.06434076364622787
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7738095238095238,
+          "precision": 0.7122033898305085,
+          "recall": 0.7294721407624634,
+          "f1": 0.719359943731317
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6428571428571429,
+          "precision": 0.5648809523809524,
+          "recall": 0.5871794871794872,
+          "f1": 0.573676603432701
+        },
+        "toxic": {
+          "n_samples": 59,
+          "accuracy": 0.9661016949152542,
+          "precision": 0.8,
+          "recall": 0.9821428571428572,
+          "f1": 0.865909090909091
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.29835153058548175,
+          "js_divergence": 0.08270536087983824
+        }
+      }
+    }
+  ],
+  "summary_stats": {
+    "size": {
+      "mse_mean": 0.9543035685322051,
+      "mse_std": 0.5981213334036609,
+      "rmse_mean": 0.9270408839286446,
+      "rmse_std": 0.3080564364803995,
+      "mae_mean": 0.5320147123966243,
+      "mae_std": 0.05351966804020229,
+      "r2_mean": 0.04726470045023875,
+      "r2_std": 0.126102268122003
+    },
+    "delivery": {
+      "mse_mean": 0.8270684480229857,
+      "mse_std": 0.17693567699093907,
+      "rmse_mean": 0.9042243053458754,
+      "rmse_std": 0.09719492602371275,
+      "mae_mean": 0.615708873112371,
+      "mae_std": 0.039477393320796444,
+      "r2_mean": 0.1574941704451898,
+      "r2_std": 0.07929093899582297
+    },
+    "pdi": {
+      "accuracy_mean": 0.7142857142857143,
+      "accuracy_std": 0.04123930494211614,
+      "precision_mean": 0.657051911797391,
+      "precision_std": 0.05438419992076548,
+      "recall_mean": 0.6835853401829536,
+      "recall_std": 0.06618289811754123,
+      "f1_mean": 0.6610780721243127,
+      "f1_std": 0.05631636693524698
+    },
+    "ee": {
+      "accuracy_mean": 0.6365079365079365,
+      "accuracy_std": 0.05715387900758358,
+      "precision_mean": 0.5754990153190521,
+      "precision_std": 0.06434539420274109,
+      "recall_mean": 0.6092582823339125,
+      "recall_std": 0.0807804696823919,
+      "f1_mean": 0.5812582234730809,
+      "f1_std": 0.0702133273457957
+    },
+    "toxic": {
+      "accuracy_mean": 0.9478174120312263,
+      "accuracy_std": 0.018643838747495024,
+      "precision_mean": 0.7425396825396825,
+      "precision_std": 0.03650310527039388,
+      "recall_mean": 0.9726010428369776,
+      "recall_std": 0.009861789747008767,
+      "f1_mean": 0.8108612462594762,
+      "f1_std": 0.038209835226095154
+    },
+    "biodist": {
+      "kl_divergence_mean": 0.3107850118699246,
+      "kl_divergence_std": 0.03838394769553977,
+      "js_divergence_mean": 0.07653345565385793,
+      "js_divergence_std": 0.012738301434340309
+    }
+  }
+}
\ No newline at end of file
diff --git a/results/pretrained_encoders/moe_pretrain_s42_summary.json b/results/pretrained_encoders/moe_pretrain_s42_summary.json
new file mode 100644
index 0000000..3dbeb97
--- /dev/null
+++ b/results/pretrained_encoders/moe_pretrain_s42_summary.json
@@ -0,0 +1,977 @@
+{
+  "fold_results": [
+    {
+      "fold": 0,
+      "best_params": {
+        "dropout": 0.4884866103742062,
+        "lr": 0.0002617457281978971,
+        "weight_decay": 9.524902838811199e-05,
+        "backbone_lr_ratio": 0.9539957173265954,
+        "moe_n_experts": 4,
+        "moe_top_k": 2,
+        "moe_expert_hidden_mult": 2,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 23,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.5279509180840005,
+          "rmse": 1.2361031179007682,
+          "mae": 0.578313840710255,
+          "r2": 0.0648564277702961
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 0.6655882063296256,
+          "rmse": 0.8158358942395374,
+          "mae": 0.674296624894286,
+          "r2": 0.15399722555537465
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7619047619047619,
+          "precision": 0.7122257053291536,
+          "recall": 0.7412687099073414,
+          "f1": 0.7214854111405835
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.7142857142857143,
+          "precision": 0.6748583260680036,
+          "recall": 0.7142857142857144,
+          "f1": 0.6773495605138607
+        },
+        "toxic": {
+          "n_samples": 58,
+          "accuracy": 0.9655172413793104,
+          "precision": 0.75,
+          "recall": 0.9821428571428572,
+          "f1": 0.8242424242424242
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.2500928345559933,
+          "js_divergence": 0.05156067014803717
+        }
+      }
+    },
+    {
+      "fold": 1,
+      "best_params": {
+        "dropout": 0.10335173439505978,
+        "lr": 0.000196757629557381,
+        "weight_decay": 0.0003725444839719432,
+        "backbone_lr_ratio": 0.05035847475715459,
+        "moe_n_experts": 4,
+        "moe_top_k": 1,
+        "moe_expert_hidden_mult": 1,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 19,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.34213962908738765,
+          "rmse": 0.584927028856923,
+          "mae": 0.45376784687063526,
+          "r2": -0.044772788836473065
+        },
+        "delivery": {
+          "n_samples": 61,
+          "mse": 1.129722640308642,
+          "rmse": 1.0628841142423016,
+          "mae": 0.6647483138825561,
+          "r2": 0.1448814017846829
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6309523809523809,
+          "precision": 0.5535714285714286,
+          "recall": 0.5634920634920635,
+          "f1": 0.553421368547419
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.5238095238095238,
+          "precision": 0.46498599439775906,
+          "recall": 0.46436896436896435,
+          "f1": 0.4483749534478767
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9508196721311475,
+          "precision": 0.75,
+          "recall": 0.9741379310344828,
+          "f1": 0.8200589970501475
+        },
+        "biodist": {
+          "n_samples": 61,
+          "kl_divergence": 0.49848379935596204,
+          "js_divergence": 0.1305599268889464
+        }
+      }
+    },
+    {
+      "fold": 2,
+      "best_params": {
+        "dropout": 0.2514825943283291,
+        "lr": 0.00029260650717186565,
+        "weight_decay": 0.0001495891705646636,
+        "backbone_lr_ratio": 0.2434609849039314,
+        "moe_n_experts": 2,
+        "moe_top_k": 2,
+        "moe_expert_hidden_mult": 1,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 20,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.3851340701995523,
+          "rmse": 0.6205917097412375,
+          "mae": 0.4943395415542736,
+          "r2": 0.2393670925336251
+        },
+        "delivery": {
+          "n_samples": 60,
+          "mse": 0.7088597182801071,
+          "rmse": 0.8419380727108776,
+          "mae": 0.5629216438857838,
+          "r2": 0.0563848526375017
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6904761904761905,
+          "precision": 0.6505882352941177,
+          "recall": 0.6876832844574781,
+          "f1": 0.6517857142857143
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6666666666666666,
+          "precision": 0.6139316239316239,
+          "recall": 0.6785714285714285,
+          "f1": 0.6277576489533011
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9344262295081968,
+          "precision": 0.7142857142857143,
+          "recall": 0.9655172413793103,
+          "f1": 0.7821428571428571
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.2665180519269797,
+          "js_divergence": 0.06450481128744694
+        }
+      }
+    },
+    {
+      "fold": 3,
+      "best_params": {
+        "dropout": 0.40463841601213524,
+        "lr": 0.0002572978619380859,
+        "weight_decay": 2.0354315009822035e-05,
+        "backbone_lr_ratio": 0.8823918523568081,
+        "moe_n_experts": 4,
+        "moe_top_k": 2,
+        "moe_expert_hidden_mult": 2,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 19,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.820237956048593,
+          "rmse": 1.3491619458199202,
+          "mae": 0.6053206579400653,
+          "r2": -0.029540461621649028
+        },
+        "delivery": {
+          "n_samples": 59,
+          "mse": 0.6693395316760985,
+          "rmse": 0.8181317324710603,
+          "mae": 0.5448598891843931,
+          "r2": 0.34932245037470455
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6785714285714286,
+          "precision": 0.6159874608150471,
+          "recall": 0.6356304985337243,
+          "f1": 0.6199095022624435
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6309523809523809,
+          "precision": 0.5653410228683567,
+          "recall": 0.5826043237807944,
+          "f1": 0.5650366426763943
+        },
+        "toxic": {
+          "n_samples": 60,
+          "accuracy": 0.9333333333333333,
+          "precision": 0.7142857142857143,
+          "recall": 0.9649122807017544,
+          "f1": 0.7818181818181817
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.29215591110500194,
+          "js_divergence": 0.07269076599614506
+        }
+      }
+    },
+    {
+      "fold": 4,
+      "best_params": {
+        "dropout": 0.3730299609537786,
+        "lr": 0.0007646220605180808,
+        "weight_decay": 0.004588258639584098,
+        "backbone_lr_ratio": 0.17129190112240483,
+        "moe_n_experts": 2,
+        "moe_top_k": 2,
+        "moe_expert_hidden_mult": 2,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 15,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.7676733172863143,
+          "rmse": 0.8761696852130381,
+          "mae": 0.5270063721386361,
+          "r2": -0.02724225576902284
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 0.9040350302083842,
+          "rmse": 0.9508075673912068,
+          "mae": 0.6287906430019387,
+          "r2": 0.12581304969093698
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7261904761904762,
+          "precision": 0.6836007130124777,
+          "recall": 0.7265395894428153,
+          "f1": 0.6891391794046662
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.7023809523809523,
+          "precision": 0.6645240097721175,
+          "recall": 0.7038461538461539,
+          "f1": 0.6728729539789446
+        },
+        "toxic": {
+          "n_samples": 59,
+          "accuracy": 0.9661016949152542,
+          "precision": 0.8,
+          "recall": 0.9821428571428572,
+          "f1": 0.865909090909091
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.20431460217894573,
+          "js_divergence": 0.05043386132176378
+        }
+      }
+    },
+    {
+      "fold": 0,
+      "best_params": {
+        "dropout": 0.4884866103742062,
+        "lr": 0.0002617457281978971,
+        "weight_decay": 9.524902838811199e-05,
+        "backbone_lr_ratio": 0.9539957173265954,
+        "moe_n_experts": 4,
+        "moe_top_k": 2,
+        "moe_expert_hidden_mult": 2,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 23,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.5550563846692986,
+          "rmse": 1.2470189993217018,
+          "mae": 0.5954494658094572,
+          "r2": 0.04826721502168674
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 0.704182837608261,
+          "rmse": 0.8391560269748772,
+          "mae": 0.6780449980548744,
+          "r2": 0.10494112024899149
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7142857142857143,
+          "precision": 0.6737967914438503,
+          "recall": 0.7084818246614397,
+          "f1": 0.6785714285714286
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.7261904761904762,
+          "precision": 0.6931286549707601,
+          "recall": 0.7380952380952381,
+          "f1": 0.7007070707070707
+        },
+        "toxic": {
+          "n_samples": 58,
+          "accuracy": 0.9655172413793104,
+          "precision": 0.75,
+          "recall": 0.9821428571428572,
+          "f1": 0.8242424242424242
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.2453488369022721,
+          "js_divergence": 0.052406074597242516
+        }
+      }
+    },
+    {
+      "fold": 1,
+      "best_params": {
+        "dropout": 0.10335173439505978,
+        "lr": 0.000196757629557381,
+        "weight_decay": 0.0003725444839719432,
+        "backbone_lr_ratio": 0.05035847475715459,
+        "moe_n_experts": 4,
+        "moe_top_k": 1,
+        "moe_expert_hidden_mult": 1,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 19,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.3727706424788934,
+          "rmse": 0.610549459486202,
+          "mae": 0.48057969708350445,
+          "r2": -0.13830901371429016
+        },
+        "delivery": {
+          "n_samples": 61,
+          "mse": 1.0803452340552668,
+          "rmse": 1.0393965720817377,
+          "mae": 0.6485996645432516,
+          "r2": 0.18225653875401804
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6309523809523809,
+          "precision": 0.5535714285714286,
+          "recall": 0.5634920634920635,
+          "f1": 0.553421368547419
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.5476190476190477,
+          "precision": 0.49737237237237236,
+          "recall": 0.5073022215879358,
+          "f1": 0.4920019687461548
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9508196721311475,
+          "precision": 0.75,
+          "recall": 0.9741379310344828,
+          "f1": 0.8200589970501475
+        },
+        "biodist": {
+          "n_samples": 61,
+          "kl_divergence": 0.5055220562600806,
+          "js_divergence": 0.1300376337593102
+        }
+      }
+    },
+    {
+      "fold": 2,
+      "best_params": {
+        "dropout": 0.2514825943283291,
+        "lr": 0.00029260650717186565,
+        "weight_decay": 0.0001495891705646636,
+        "backbone_lr_ratio": 0.2434609849039314,
+        "moe_n_experts": 2,
+        "moe_top_k": 2,
+        "moe_expert_hidden_mult": 1,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 20,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.501474748290053,
+          "rmse": 0.708148817897801,
+          "mae": 0.5538181106981245,
+          "r2": 0.009596331959947801
+        },
+        "delivery": {
+          "n_samples": 60,
+          "mse": 0.739867919930154,
+          "rmse": 0.8601557532971305,
+          "mae": 0.5739390922787909,
+          "r2": 0.015107561778814449
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7380952380952381,
+          "precision": 0.6923076923076923,
+          "recall": 0.7346041055718475,
+          "f1": 0.6998050682261208
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6785714285714286,
+          "precision": 0.6228947368421053,
+          "recall": 0.6851073762838468,
+          "f1": 0.63836444060848
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9344262295081968,
+          "precision": 0.7142857142857143,
+          "recall": 0.9655172413793103,
+          "f1": 0.7821428571428571
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.2914659417557568,
+          "js_divergence": 0.07149076417555128
+        }
+      }
+    },
+    {
+      "fold": 3,
+      "best_params": {
+        "dropout": 0.40463841601213524,
+        "lr": 0.0002572978619380859,
+        "weight_decay": 2.0354315009822035e-05,
+        "backbone_lr_ratio": 0.8823918523568081,
+        "moe_n_experts": 4,
+        "moe_top_k": 2,
+        "moe_expert_hidden_mult": 2,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 19,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.7508482794444888,
+          "rmse": 1.3231962361813492,
+          "mae": 0.5995979247704507,
+          "r2": 0.009706868347147068
+        },
+        "delivery": {
+          "n_samples": 59,
+          "mse": 0.6800398997076931,
+          "rmse": 0.824645317520019,
+          "mae": 0.5383539610084588,
+          "r2": 0.33892042132757505
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7023809523809523,
+          "precision": 0.6423197492163009,
+          "recall": 0.6664222873900294,
+          "f1": 0.6480643539467068
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6666666666666666,
+          "precision": 0.5898730898730898,
+          "recall": 0.6022121669180492,
+          "f1": 0.5907255864702673
+        },
+        "toxic": {
+          "n_samples": 60,
+          "accuracy": 0.9166666666666666,
+          "precision": 0.6875,
+          "recall": 0.956140350877193,
+          "f1": 0.749791492910759
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.22697074487882685,
+          "js_divergence": 0.05572372821276913
+        }
+      }
+    },
+    {
+      "fold": 4,
+      "best_params": {
+        "dropout": 0.3730299609537786,
+        "lr": 0.0007646220605180808,
+        "weight_decay": 0.004588258639584098,
+        "backbone_lr_ratio": 0.17129190112240483,
+        "moe_n_experts": 2,
+        "moe_top_k": 2,
+        "moe_expert_hidden_mult": 2,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 15,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.7388933186856751,
+          "rmse": 0.8595890405802502,
+          "mae": 0.4949821270059883,
+          "r2": 0.01126895729235744
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 1.0735857604041814,
+          "rmse": 1.0361398363175607,
+          "mae": 0.6398222603160759,
+          "r2": -0.038139707447659266
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7023809523809523,
+          "precision": 0.6345029239766082,
+          "recall": 0.6517595307917888,
+          "f1": 0.6398559423769508
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.7261904761904762,
+          "precision": 0.662962962962963,
+          "recall": 0.7064102564102565,
+          "f1": 0.6789544933874829
+        },
+        "toxic": {
+          "n_samples": 59,
+          "accuracy": 0.9661016949152542,
+          "precision": 0.8,
+          "recall": 0.9821428571428572,
+          "f1": 0.865909090909091
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.2849032950499658,
+          "js_divergence": 0.0742269697587313
+        }
+      }
+    },
+    {
+      "fold": 0,
+      "best_params": {
+        "dropout": 0.4884866103742062,
+        "lr": 0.0002617457281978971,
+        "weight_decay": 9.524902838811199e-05,
+        "backbone_lr_ratio": 0.9539957173265954,
+        "moe_n_experts": 4,
+        "moe_top_k": 2,
+        "moe_expert_hidden_mult": 2,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 23,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.575393168842647,
+          "rmse": 1.255146672242988,
+          "mae": 0.5922032207340361,
+          "r2": 0.035820602519645584
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 0.6805529919450708,
+          "rmse": 0.8249563576002495,
+          "mae": 0.6910147559026192,
+          "r2": 0.13497608000435835
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7976190476190477,
+          "precision": 0.75,
+          "recall": 0.7794012829650748,
+          "f1": 0.7606837606837606
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.7380952380952381,
+          "precision": 0.7064443475733798,
+          "recall": 0.7547619047619047,
+          "f1": 0.7126436781609194
+        },
+        "toxic": {
+          "n_samples": 58,
+          "accuracy": 0.9655172413793104,
+          "precision": 0.75,
+          "recall": 0.9821428571428572,
+          "f1": 0.8242424242424242
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.23590056396282402,
+          "js_divergence": 0.047894235451583785
+        }
+      }
+    },
+    {
+      "fold": 1,
+      "best_params": {
+        "dropout": 0.10335173439505978,
+        "lr": 0.000196757629557381,
+        "weight_decay": 0.0003725444839719432,
+        "backbone_lr_ratio": 0.05035847475715459,
+        "moe_n_experts": 4,
+        "moe_top_k": 1,
+        "moe_expert_hidden_mult": 1,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 19,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.3579244858231685,
+          "rmse": 0.5982679047242703,
+          "mae": 0.46379138462777647,
+          "r2": -0.09297412943304506
+        },
+        "delivery": {
+          "n_samples": 61,
+          "mse": 1.1096558552074713,
+          "rmse": 1.0534020387332992,
+          "mae": 0.6546663998885721,
+          "r2": 0.1600705115131673
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6547619047619048,
+          "precision": 0.5696286472148542,
+          "recall": 0.5793650793650793,
+          "f1": 0.571654650958326
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.5476190476190477,
+          "precision": 0.49253034547152197,
+          "recall": 0.49681271109842545,
+          "f1": 0.4801587301587302
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9344262295081968,
+          "precision": 0.7142857142857143,
+          "recall": 0.9655172413793103,
+          "f1": 0.7821428571428571
+        },
+        "biodist": {
+          "n_samples": 61,
+          "kl_divergence": 0.502867985249319,
+          "js_divergence": 0.13123665503825419
+        }
+      }
+    },
+    {
+      "fold": 2,
+      "best_params": {
+        "dropout": 0.2514825943283291,
+        "lr": 0.00029260650717186565,
+        "weight_decay": 0.0001495891705646636,
+        "backbone_lr_ratio": 0.2434609849039314,
+        "moe_n_experts": 2,
+        "moe_top_k": 2,
+        "moe_expert_hidden_mult": 1,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 20,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.618457534400238,
+          "rmse": 0.7864207108159335,
+          "mae": 0.5679304331563235,
+          "r2": -0.22144258048008303
+        },
+        "delivery": {
+          "n_samples": 60,
+          "mse": 0.7362245815365097,
+          "rmse": 0.8580353031994137,
+          "mae": 0.5702473803190514,
+          "r2": 0.019957476658378215
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6785714285714286,
+          "precision": 0.6523864289821737,
+          "recall": 0.6942815249266863,
+          "f1": 0.6473332296687918
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6785714285714286,
+          "precision": 0.6228947368421053,
+          "recall": 0.6851073762838468,
+          "f1": 0.63836444060848
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9344262295081968,
+          "precision": 0.7142857142857143,
+          "recall": 0.9655172413793103,
+          "f1": 0.7821428571428571
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.26356343521196823,
+          "js_divergence": 0.06378606399643424
+        }
+      }
+    },
+    {
+      "fold": 3,
+      "best_params": {
+        "dropout": 0.40463841601213524,
+        "lr": 0.0002572978619380859,
+        "weight_decay": 2.0354315009822035e-05,
+        "backbone_lr_ratio": 0.8823918523568081,
+        "moe_n_experts": 4,
+        "moe_top_k": 2,
+        "moe_expert_hidden_mult": 2,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 19,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.8408361971730496,
+          "rmse": 1.3567741879815702,
+          "mae": 0.6257544057668153,
+          "r2": -0.04119098379947639
+        },
+        "delivery": {
+          "n_samples": 59,
+          "mse": 0.737776343129472,
+          "rmse": 0.8589390799873249,
+          "mae": 0.5497218715026975,
+          "r2": 0.28279373860245827
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6904761904761905,
+          "precision": 0.6333333333333333,
+          "recall": 0.6583577712609971,
+          "f1": 0.6379310344827587
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6428571428571429,
+          "precision": 0.5640697674418605,
+          "recall": 0.5891402714932127,
+          "f1": 0.5710279611967066
+        },
+        "toxic": {
+          "n_samples": 60,
+          "accuracy": 0.9333333333333333,
+          "precision": 0.7142857142857143,
+          "recall": 0.9649122807017544,
+          "f1": 0.7818181818181817
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.24488500401793606,
+          "js_divergence": 0.060842241588791174
+        }
+      }
+    },
+    {
+      "fold": 4,
+      "best_params": {
+        "dropout": 0.3730299609537786,
+        "lr": 0.0007646220605180808,
+        "weight_decay": 0.004588258639584098,
+        "backbone_lr_ratio": 0.17129190112240483,
+        "moe_n_experts": 2,
+        "moe_top_k": 2,
+        "moe_expert_hidden_mult": 2,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 15,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.712297885673984,
+          "rmse": 0.8439774201209319,
+          "mae": 0.5084991393599748,
+          "r2": 0.046856950237922024
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 1.1055967310671657,
+          "rmse": 1.0514735997956228,
+          "mae": 0.6599975297677105,
+          "r2": -0.06909378763839769
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7619047619047619,
+          "precision": 0.711111111111111,
+          "recall": 0.750733137829912,
+          "f1": 0.7214854111405835
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.7023809523809523,
+          "precision": 0.6536363636363637,
+          "recall": 0.7038461538461539,
+          "f1": 0.6717592592592593
+        },
+        "toxic": {
+          "n_samples": 59,
+          "accuracy": 0.9661016949152542,
+          "precision": 0.8,
+          "recall": 0.9821428571428572,
+          "f1": 0.865909090909091
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.1782104997564988,
+          "js_divergence": 0.04297224929274232
+        }
+      }
+    }
+  ],
+  "summary_stats": {
+    "size": {
+      "mse_mean": 0.9911392357458229,
+      "mse_std": 0.5817120308358389,
+      "rmse_mean": 0.9504028624589923,
+      "rmse_std": 0.29643487442535593,
+      "mae_mean": 0.542756944548421,
+      "mae_std": 0.054981488862431024,
+      "r2_mean": -0.008648784531427449,
+      "r2_std": 0.09954199193808574
+    },
+    "delivery": {
+      "mse_mean": 0.8483582187596068,
+      "mse_std": 0.18619352158793398,
+      "rmse_mean": 0.9157264844374813,
+      "rmse_std": 0.09901123400593626,
+      "mae_mean": 0.6186683352287373,
+      "mae_std": 0.05324906806818642,
+      "r2_mean": 0.13081259558966032,
+      "r2_std": 0.1207826593867956
+    },
+    "pdi": {
+      "accuracy_mean": 0.703968253968254,
+      "accuracy_std": 0.04618202071738462,
+      "precision_mean": 0.6485954432786384,
+      "precision_std": 0.056485137303304574,
+      "recall_mean": 0.6761008502725561,
+      "recall_std": 0.06599285309996766,
+      "f1_mean": 0.6529698282829115,
+      "f1_std": 0.05951390665637187
+    },
+    "ee": {
+      "accuracy_mean": 0.6595238095238095,
+      "accuracy_std": 0.0668930397006523,
+      "precision_mean": 0.6059632236682923,
+      "precision_std": 0.07313293968503662,
+      "recall_mean": 0.640831484108795,
+      "recall_std": 0.0904387508717925,
+      "f1_mean": 0.6110732925915952,
+      "f1_std": 0.08105368908168387
+    },
+    "toxic": {
+      "accuracy_mean": 0.947835626967474,
+      "accuracy_std": 0.01646827276703399,
+      "precision_mean": 0.741547619047619,
+      "precision_std": 0.034629149636327315,
+      "recall_mean": 0.9726111255149369,
+      "recall_std": 0.008721274087923442,
+      "f1_mean": 0.810171454978226,
+      "f1_std": 0.03512408564460891
+    },
+    "biodist": {
+      "kl_divergence_mean": 0.299413570811222,
+      "kl_divergence_std": 0.1056856517385418,
+      "js_divergence_mean": 0.0733577767675833,
+      "js_divergence_std": 0.029986885602095156
+    }
+  }
+}
\ No newline at end of file
diff --git a/results/pretrained_encoders/molformer_pretrain_s42_summary.json b/results/pretrained_encoders/molformer_pretrain_s42_summary.json
new file mode 100644
index 0000000..74d9b9d
--- /dev/null
+++ b/results/pretrained_encoders/molformer_pretrain_s42_summary.json
@@ -0,0 +1,932 @@
+{
+  "fold_results": [
+    {
+      "fold": 0,
+      "best_params": {
+        "dropout": 0.3980773018180746,
+        "lr": 0.0009896030589860336,
+        "weight_decay": 0.0002988554096805342,
+        "backbone_lr_ratio": 0.2882797738700643,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 13,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.6246690728469169,
+          "rmse": 1.274625071480597,
+          "mae": 0.5944420359916536,
+          "r2": 0.005662536347478486
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 0.7102819707127421,
+          "rmse": 0.842782279543621,
+          "mae": 0.6625874312594533,
+          "r2": 0.09718875402761229
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.8333333333333334,
+          "precision": 0.7891525423728813,
+          "recall": 0.8039914468995011,
+          "f1": 0.7958333333333334
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.7142857142857143,
+          "precision": 0.6829362569400521,
+          "recall": 0.7314285714285714,
+          "f1": 0.687769981711114
+        },
+        "toxic": {
+          "n_samples": 58,
+          "accuracy": 0.9655172413793104,
+          "precision": 0.75,
+          "recall": 0.9821428571428572,
+          "f1": 0.8242424242424242
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.32097759110945434,
+          "js_divergence": 0.06712981080576393
+        }
+      }
+    },
+    {
+      "fold": 1,
+      "best_params": {
+        "dropout": 0.396946460618591,
+        "lr": 0.0004854064403598687,
+        "weight_decay": 0.00010992238518795333,
+        "backbone_lr_ratio": 0.23864875345272699,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 10,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.38773907280523073,
+          "rmse": 0.6226869781882634,
+          "mae": 0.478045186826161,
+          "r2": -0.18401727831452308
+        },
+        "delivery": {
+          "n_samples": 61,
+          "mse": 1.1492300175260897,
+          "rmse": 1.0720214631834988,
+          "mae": 0.7018961062746458,
+          "r2": 0.13011572349706002
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.5952380952380952,
+          "precision": 0.5664174813110983,
+          "recall": 0.5873015873015872,
+          "f1": 0.5523510971786834
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.5476190476190477,
+          "precision": 0.4826358616681197,
+          "recall": 0.4884639170353456,
+          "f1": 0.47545555874980455
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9344262295081968,
+          "precision": 0.7142857142857143,
+          "recall": 0.9655172413793103,
+          "f1": 0.7821428571428571
+        },
+        "biodist": {
+          "n_samples": 61,
+          "kl_divergence": 0.4828981973271936,
+          "js_divergence": 0.12323903696473625
+        }
+      }
+    },
+    {
+      "fold": 2,
+      "best_params": {
+        "dropout": 0.10827786083228613,
+        "lr": 0.0004876914007718381,
+        "weight_decay": 0.0852885971769344,
+        "backbone_lr_ratio": 0.21714091995810716,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 8,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.4123269005771989,
+          "rmse": 0.6421268570751413,
+          "mae": 0.49745388741471935,
+          "r2": 0.1856617383911774
+        },
+        "delivery": {
+          "n_samples": 60,
+          "mse": 0.6503294595263629,
+          "rmse": 0.8064300710702466,
+          "mae": 0.5487457474150385,
+          "r2": 0.1342987717315124
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.5595238095238095,
+          "precision": 0.5652297220646625,
+          "recall": 0.5843108504398826,
+          "f1": 0.5357729648991785
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6547619047619048,
+          "precision": 0.6053846153846153,
+          "recall": 0.6720354808590102,
+          "f1": 0.6203502415458937
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9344262295081968,
+          "precision": 0.7142857142857143,
+          "recall": 0.9655172413793103,
+          "f1": 0.7821428571428571
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.3376426852856137,
+          "js_divergence": 0.09061480035897176
+        }
+      }
+    },
+    {
+      "fold": 3,
+      "best_params": {
+        "dropout": 0.1463802845991778,
+        "lr": 0.00017979738268552602,
+        "weight_decay": 0.026590603512062855,
+        "backbone_lr_ratio": 0.19962036354162335,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 18,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.7510092489615507,
+          "rmse": 1.3232570608016987,
+          "mae": 0.6278676227834749,
+          "r2": 0.009615822761402493
+        },
+        "delivery": {
+          "n_samples": 59,
+          "mse": 0.7222390124885935,
+          "rmse": 0.84984646406783,
+          "mae": 0.57879668119032,
+          "r2": 0.29789786998972123
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7380952380952381,
+          "precision": 0.6785714285714286,
+          "recall": 0.7052785923753666,
+          "f1": 0.6867796610169492
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.5476190476190477,
+          "precision": 0.5483829895594602,
+          "recall": 0.575062845651081,
+          "f1": 0.5199383832295225
+        },
+        "toxic": {
+          "n_samples": 60,
+          "accuracy": 0.9333333333333333,
+          "precision": 0.7142857142857143,
+          "recall": 0.9649122807017544,
+          "f1": 0.7818181818181817
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.4613132992323466,
+          "js_divergence": 0.12165500439647682
+        }
+      }
+    },
+    {
+      "fold": 4,
+      "best_params": {
+        "dropout": 0.35317424690388177,
+        "lr": 0.0009765192272158632,
+        "weight_decay": 0.00012175015526770499,
+        "backbone_lr_ratio": 0.023808704109897987,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 18,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.6604880002050739,
+          "rmse": 0.8127041283302762,
+          "mae": 0.4948491998892721,
+          "r2": 0.11618501260741076
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 0.936642398150463,
+          "rmse": 0.9678028715345202,
+          "mae": 0.6353725126848139,
+          "r2": 0.09428226317670119
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6071428571428571,
+          "precision": 0.6374353076480737,
+          "recall": 0.6752199413489737,
+          "f1": 0.5942028985507246
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6904761904761905,
+          "precision": 0.6475761475761476,
+          "recall": 0.7401709401709402,
+          "f1": 0.6653096672637463
+        },
+        "toxic": {
+          "n_samples": 59,
+          "accuracy": 0.9661016949152542,
+          "precision": 0.8,
+          "recall": 0.9821428571428572,
+          "f1": 0.865909090909091
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.40727903738537147,
+          "js_divergence": 0.10249950448751813
+        }
+      }
+    },
+    {
+      "fold": 0,
+      "best_params": {
+        "dropout": 0.3980773018180746,
+        "lr": 0.0009896030589860336,
+        "weight_decay": 0.0002988554096805342,
+        "backbone_lr_ratio": 0.2882797738700643,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 13,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.545312966192772,
+          "rmse": 1.2431061765564404,
+          "mae": 0.5886967062770602,
+          "r2": 0.054230426962613154
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 0.6668586486255093,
+          "rmse": 0.8166141369248449,
+          "mae": 0.6276618008790859,
+          "r2": 0.15238241673384167
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.8452380952380952,
+          "precision": 0.8041666666666667,
+          "recall": 0.8121881682109764,
+          "f1": 0.8079831193951117
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.7261904761904762,
+          "precision": 0.6914328121224673,
+          "recall": 0.7452380952380953,
+          "f1": 0.701447533036203
+        },
+        "toxic": {
+          "n_samples": 58,
+          "accuracy": 0.9827586206896551,
+          "precision": 0.9912280701754386,
+          "recall": 0.75,
+          "f1": 0.8289085545722714
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.27259534788302564,
+          "js_divergence": 0.06020453275369037
+        }
+      }
+    },
+    {
+      "fold": 1,
+      "best_params": {
+        "dropout": 0.396946460618591,
+        "lr": 0.0004854064403598687,
+        "weight_decay": 0.00010992238518795333,
+        "backbone_lr_ratio": 0.23864875345272699,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 10,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.4005803081504934,
+          "rmse": 0.6329141396354591,
+          "mae": 0.4951437964946741,
+          "r2": -0.2232298457086057
+        },
+        "delivery": {
+          "n_samples": 61,
+          "mse": 1.1667453321847503,
+          "rmse": 1.0801598641797197,
+          "mae": 0.6814669590809794,
+          "r2": 0.11685789296077709
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.5595238095238095,
+          "precision": 0.5477272727272727,
+          "recall": 0.5634920634920635,
+          "f1": 0.5238241152137275
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.5238095238095238,
+          "precision": 0.456992337164751,
+          "recall": 0.46650968079539507,
+          "f1": 0.45625740933077347
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9344262295081968,
+          "precision": 0.7142857142857143,
+          "recall": 0.9655172413793103,
+          "f1": 0.7821428571428571
+        },
+        "biodist": {
+          "n_samples": 61,
+          "kl_divergence": 0.5730307313158203,
+          "js_divergence": 0.14005679605749516
+        }
+      }
+    },
+    {
+      "fold": 2,
+      "best_params": {
+        "dropout": 0.10827786083228613,
+        "lr": 0.0004876914007718381,
+        "weight_decay": 0.0852885971769344,
+        "backbone_lr_ratio": 0.21714091995810716,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 8,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.45499801818836005,
+          "rmse": 0.6745354091434785,
+          "mae": 0.5309686183796397,
+          "r2": 0.1013870435125862
+        },
+        "delivery": {
+          "n_samples": 60,
+          "mse": 0.6030715996767626,
+          "rmse": 0.7765768472448574,
+          "mae": 0.5282929584461574,
+          "r2": 0.1972071741079553
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6785714285714286,
+          "precision": 0.6336898395721925,
+          "recall": 0.6649560117302054,
+          "f1": 0.635076427996782
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6547619047619048,
+          "precision": 0.5992402659069326,
+          "recall": 0.6507936507936507,
+          "f1": 0.6101851851851852
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9344262295081968,
+          "precision": 0.7142857142857143,
+          "recall": 0.9655172413793103,
+          "f1": 0.7821428571428571
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.358983754945942,
+          "js_divergence": 0.09298507446003766
+        }
+      }
+    },
+    {
+      "fold": 3,
+      "best_params": {
+        "dropout": 0.1463802845991778,
+        "lr": 0.00017979738268552602,
+        "weight_decay": 0.026590603512062855,
+        "backbone_lr_ratio": 0.19962036354162335,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 18,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.6469751588239248,
+          "rmse": 1.2833452999188975,
+          "mae": 0.6211532356262387,
+          "r2": 0.06845829708118345
+        },
+        "delivery": {
+          "n_samples": 59,
+          "mse": 0.7509799772948998,
+          "rmse": 0.866591009239595,
+          "mae": 0.6103691533050042,
+          "r2": 0.2699582374579258
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6428571428571429,
+          "precision": 0.6545454545454545,
+          "recall": 0.6994134897360704,
+          "f1": 0.625668449197861
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6190476190476191,
+          "precision": 0.5681642512077295,
+          "recall": 0.6041478129713425,
+          "f1": 0.563984674329502
+        },
+        "toxic": {
+          "n_samples": 60,
+          "accuracy": 0.9333333333333333,
+          "precision": 0.7142857142857143,
+          "recall": 0.9649122807017544,
+          "f1": 0.7818181818181817
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.4523145833037008,
+          "js_divergence": 0.1183277351587518
+        }
+      }
+    },
+    {
+      "fold": 4,
+      "best_params": {
+        "dropout": 0.35317424690388177,
+        "lr": 0.0009765192272158632,
+        "weight_decay": 0.00012175015526770499,
+        "backbone_lr_ratio": 0.023808704109897987,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 18,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.6556117434601818,
+          "rmse": 0.8096985509806608,
+          "mae": 0.5088372581716006,
+          "r2": 0.12271004984074696
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 1.0003346592105318,
+          "rmse": 1.0001673156080095,
+          "mae": 0.6612379994114925,
+          "r2": 0.03269289816995369
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.7142857142857143,
+          "precision": 0.6592592592592592,
+          "recall": 0.6891495601173021,
+          "f1": 0.6657824933687002
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6428571428571429,
+          "precision": 0.6072310405643738,
+          "recall": 0.6940170940170941,
+          "f1": 0.6169253712734788
+        },
+        "toxic": {
+          "n_samples": 59,
+          "accuracy": 0.9661016949152542,
+          "precision": 0.8,
+          "recall": 0.9821428571428572,
+          "f1": 0.865909090909091
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.4083994319724002,
+          "js_divergence": 0.1006941777950375
+        }
+      }
+    },
+    {
+      "fold": 0,
+      "best_params": {
+        "dropout": 0.3980773018180746,
+        "lr": 0.0009896030589860336,
+        "weight_decay": 0.0002988554096805342,
+        "backbone_lr_ratio": 0.2882797738700643,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 13,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.444963857475926,
+          "rmse": 1.2020664946149717,
+          "mae": 0.575902431103659,
+          "r2": 0.11564655158081194
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 0.6487752159789035,
+          "rmse": 0.8054658378720376,
+          "mae": 0.6530212075288954,
+          "r2": 0.1753675508528415
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.8333333333333334,
+          "precision": 0.7936507936507937,
+          "recall": 0.7769066286528867,
+          "f1": 0.7844574780058651
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.7261904761904762,
+          "precision": 0.6868692000270947,
+          "recall": 0.7452380952380953,
+          "f1": 0.6987547892720306
+        },
+        "toxic": {
+          "n_samples": 58,
+          "accuracy": 0.9482758620689655,
+          "precision": 0.7,
+          "recall": 0.9732142857142857,
+          "f1": 0.7719528178243775
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.32634100712155006,
+          "js_divergence": 0.06706463335858984
+        }
+      }
+    },
+    {
+      "fold": 1,
+      "best_params": {
+        "dropout": 0.396946460618591,
+        "lr": 0.0004854064403598687,
+        "weight_decay": 0.00010992238518795333,
+        "backbone_lr_ratio": 0.23864875345272699,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 10,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.45034502529074416,
+          "rmse": 0.6710775106429541,
+          "mae": 0.5225315168943434,
+          "r2": -0.37519359936953656
+        },
+        "delivery": {
+          "n_samples": 61,
+          "mse": 1.1274144896048421,
+          "rmse": 1.0617977630438116,
+          "mae": 0.6428290372561725,
+          "r2": 0.14662850547534068
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6904761904761905,
+          "precision": 0.5972915181753385,
+          "recall": 0.6031746031746031,
+          "f1": 0.5997067448680352
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.5238095238095238,
+          "precision": 0.4141642671054436,
+          "recall": 0.42669235526378385,
+          "f1": 0.40981240981240985
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9016393442622951,
+          "precision": 0.6666666666666666,
+          "recall": 0.9482758620689655,
+          "f1": 0.7227272727272727
+        },
+        "biodist": {
+          "n_samples": 61,
+          "kl_divergence": 0.5404966442274723,
+          "js_divergence": 0.13248730719985696
+        }
+      }
+    },
+    {
+      "fold": 2,
+      "best_params": {
+        "dropout": 0.10827786083228613,
+        "lr": 0.0004876914007718381,
+        "weight_decay": 0.0852885971769344,
+        "backbone_lr_ratio": 0.21714091995810716,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 8,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.5404522607814335,
+          "rmse": 0.7351545829153441,
+          "mae": 0.5627435291618375,
+          "r2": -0.06738355880059732
+        },
+        "delivery": {
+          "n_samples": 60,
+          "mse": 0.6347980231433975,
+          "rmse": 0.7967421308951834,
+          "mae": 0.5446047547040507,
+          "r2": 0.1549738055263823
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.5952380952380952,
+          "precision": 0.6068181818181818,
+          "recall": 0.6378299120234604,
+          "f1": 0.5757575757575757
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6190476190476191,
+          "precision": 0.5553467000835423,
+          "recall": 0.588702147525677,
+          "f1": 0.5573950928686081
+        },
+        "toxic": {
+          "n_samples": 61,
+          "accuracy": 0.9344262295081968,
+          "precision": 0.7142857142857143,
+          "recall": 0.9655172413793103,
+          "f1": 0.7821428571428571
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.34103334231152865,
+          "js_divergence": 0.0899550896776649
+        }
+      }
+    },
+    {
+      "fold": 3,
+      "best_params": {
+        "dropout": 0.1463802845991778,
+        "lr": 0.00017979738268552602,
+        "weight_decay": 0.026590603512062855,
+        "backbone_lr_ratio": 0.19962036354162335,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 18,
+      "test_metrics": {
+        "size": {
+          "n_samples": 83,
+          "mse": 1.633019754709887,
+          "rmse": 1.2778966134667886,
+          "mae": 0.6318143501271865,
+          "r2": 0.07635158001484588
+        },
+        "delivery": {
+          "n_samples": 59,
+          "mse": 0.778873610785293,
+          "rmse": 0.8825381639256702,
+          "mae": 0.591008120058578,
+          "r2": 0.2428423116373991
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6785714285714286,
+          "precision": 0.5983050847457627,
+          "recall": 0.6063049853372434,
+          "f1": 0.6011957095129242
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6428571428571429,
+          "precision": 0.5736842105263158,
+          "recall": 0.6070889894419307,
+          "f1": 0.5785510785510786
+        },
+        "toxic": {
+          "n_samples": 60,
+          "accuracy": 0.9333333333333333,
+          "precision": 0.7142857142857143,
+          "recall": 0.9649122807017544,
+          "f1": 0.7818181818181817
+        },
+        "biodist": {
+          "n_samples": 60,
+          "kl_divergence": 0.46990571185549446,
+          "js_divergence": 0.12454201663905487
+        }
+      }
+    },
+    {
+      "fold": 4,
+      "best_params": {
+        "dropout": 0.35317424690388177,
+        "lr": 0.0009765192272158632,
+        "weight_decay": 0.00012175015526770499,
+        "backbone_lr_ratio": 0.023808704109897987,
+        "d_model": 256,
+        "num_heads": 8,
+        "n_attn_layers": 4,
+        "fusion_strategy": "attention",
+        "head_hidden_dim": 128,
+        "set_transformer_block": "sab"
+      },
+      "epoch_mean": 18,
+      "test_metrics": {
+        "size": {
+          "n_samples": 84,
+          "mse": 0.7197383052375195,
+          "rmse": 0.8483739182916454,
+          "mae": 0.5446056428176927,
+          "r2": 0.0369007446434223
+        },
+        "delivery": {
+          "n_samples": 58,
+          "mse": 1.0250908773516882,
+          "rmse": 1.0124677166960379,
+          "mae": 0.6531032593599682,
+          "r2": 0.008754043905628395
+        },
+        "pdi": {
+          "n_samples": 84,
+          "accuracy": 0.6190476190476191,
+          "precision": 0.6079545454545454,
+          "recall": 0.6392961876832844,
+          "f1": 0.5909920876445527
+        },
+        "ee": {
+          "n_samples": 84,
+          "accuracy": 0.6785714285714286,
+          "precision": 0.6275303643724697,
+          "recall": 0.7021367521367522,
+          "f1": 0.6415056360708534
+        },
+        "toxic": {
+          "n_samples": 59,
+          "accuracy": 0.9661016949152542,
+          "precision": 0.8,
+          "recall": 0.9821428571428572,
+          "f1": 0.865909090909091
+        },
+        "biodist": {
+          "n_samples": 58,
+          "kl_divergence": 0.40077254800841094,
+          "js_divergence": 0.10315238600509395
+        }
+      }
+    }
+  ],
+  "summary_stats": {
+    "size": {
+      "mse_mean": 0.9552153129138143,
+      "mse_std": 0.544124280885305,
+      "rmse_mean": 0.9369045861361744,
+      "rmse_std": 0.27825367812630597,
+      "mae_mean": 0.5516703345306142,
+      "mae_std": 0.050933508755883206,
+      "r2_mean": 0.0028657014366944233,
+      "r2_std": 0.1484374631238745
+    },
+    "delivery": {
+      "mse_mean": 0.8381110194840553,
+      "mse_std": 0.19956001015608432,
+      "rmse_mean": 0.9092002623352989,
+      "rmse_std": 0.1070789542976535,
+      "mae_mean": 0.6213995819236436,
+      "mae_std": 0.05084304982304164,
+      "r2_mean": 0.15009654795004349,
+      "r2_std": 0.07708969134269554
+    },
+    "pdi": {
+      "accuracy_mean": 0.6793650793650793,
+      "accuracy_std": 0.09408695848584789,
+      "precision_mean": 0.6493476732389074,
+      "precision_std": 0.08116705378595848,
+      "recall_mean": 0.6699209352348937,
+      "recall_std": 0.07666166785670459,
+      "f1_mean": 0.6383589437293336,
+      "f1_std": 0.08952118234954987
+    },
+    "ee": {
+      "accuracy_mean": 0.6341269841269842,
+      "accuracy_std": 0.06796728274922742,
+      "precision_mean": 0.583171421347301,
+      "precision_std": 0.08031795761416705,
+      "recall_mean": 0.6291817619044511,
+      "recall_std": 0.10148518503800956,
+      "f1_mean": 0.5869095341486802,
+      "f1_std": 0.08729125759225413
+    },
+    "toxic": {
+      "accuracy_mean": 0.9445751533791317,
+      "accuracy_std": 0.020063303116255492,
+      "precision_mean": 0.7481453634085212,
+      "precision_std": 0.07540464740954501,
+      "recall_mean": 0.9548256416904332,
+      "recall_std": 0.0554955125846467,
+      "f1_mean": 0.8001151448841631,
+      "f1_std": 0.039832204457600114
+    },
+    "biodist": {
+      "kl_divergence_mean": 0.41026559421902165,
+      "kl_divergence_std": 0.08303708399525898,
+      "js_divergence_mean": 0.10230719374124933,
+      "js_divergence_std": 0.023848212584556397
+    }
+  }
+}
\ No newline at end of file
diff --git a/scripts_run/encode_embeddings/README.md b/scripts_run/encode_embeddings/README.md
new file mode 100644
index 0000000..c7d5b74
--- /dev/null
+++ b/scripts_run/encode_embeddings/README.md
@@ -0,0 +1,43 @@
+# 离线预训练分子 embedding
+
+三个预训练编码器作为额外的 chemical token 接入模型。embedding 离线生成、
+存为 `.npy`（按 csv 行顺序对齐），训练时通过 `{smiles: vector}` 查表注入。
+
+npy 文件与预训练权重不入库，运行下列脚本可复现。
+
+## MolFormer (768d)
+`ibm/MoLFormer-XL-both-10pct`，HuggingFace 直接加载。
+注意：需 transformers>=4.57（依赖 `transformers.masking_utils`），
+与项目 pin 的 4.45 冲突，建议装到独立目录后用 sys.path 隔离。
+见 `encode_molformer.py`。
+
+## GraphMVP (300d)
+ICLR'22，5 层 GIN + OGB AtomEncoder/BondEncoder。
+权重：https://github.com/chao1224/GraphMVP → Drive 文件夹
+`GraphMVP_complate_features_for_regression/GraphMVP/pretraining_model.pth`
+注意：新版 OGB 的 chirality 类别数为 5（checkpoint 为 4），加载时按形状逐层对齐。
+见 `encode_graphmvp.py`。
+
+## GROVER (3200d = atom 1600 + bond 1600)
+NeurIPS'20，dualtrans (hidden 800, depth 6)。
+代码：https://github.com/tencent-ailab/grover，权重 grover_base。
+原项目基于 Python 3.6.8 / PyTorch 1.1，在现代环境需两处补丁：
+  1. `grover/util/utils.py` 的 `torch.load` 加 `weights_only=False`
+  2. `build_model` 前用 checkpoint 自带 args 补齐 current_args 缺失字段，
+     并为 dropout 等推理期字段提供默认值
+见 `encode_grover.py`。
+
+## 使用
+```bash
+python -m lnp_ml.modeling.pretrain main --device cuda \
+  --output-dir models/pretrain/<name> \
+  --<name>-emb data/interim/<name>_emb_external.npy \
+  --<name>-csv data/external/all_data_LiON.csv
+
+python -m lnp_ml.modeling.nested_cv_optuna \
+  --device cuda --n-trials 15 --epochs-per-trial 30 --n-repeats 3 --seed 42 \
+  --<name>-emb data/interim/<name>_emb.npy \
+  --<name>-csv data/interim/internal.csv \
+  --init-from-pretrain models/pretrain/<name>/pretrain_delivery.pt \
+  --output-dir models/full/<name>_s42
+```
diff --git a/scripts_run/encode_embeddings/encode_graphmvp.py b/scripts_run/encode_embeddings/encode_graphmvp.py
new file mode 100644
index 0000000..477fa5f
--- /dev/null
+++ b/scripts_run/encode_embeddings/encode_graphmvp.py
@@ -0,0 +1,122 @@
+"""GraphMVP 离线 embedding (300d)
+
+依赖：torch-geometric, ogb
+权重：https://github.com/chao1224/GraphMVP → Drive 文件夹内
+      GraphMVP_complate_features_for_regression/GraphMVP/pretraining_model.pth
+
+注意：新版 OGB 的 chirality 类别数为 5，而 checkpoint 为 4，
+      加载时按形状逐层对齐（前 4 行用预训练，新增类别保持随机初始化）。
+
+用法：
+    python encode_graphmvp.py --ckpt pretraining_model.pth \
+        --csv data/interim/internal.csv --out data/interim/graphmvp_emb.npy
+"""
+import argparse
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from ogb.graphproppred.mol_encoder import AtomEncoder, BondEncoder
+from ogb.utils.mol import smiles2graph
+from torch_geometric.data import Batch, Data
+from torch_geometric.nn import MessagePassing, global_mean_pool
+
+
+class GINConv(MessagePassing):
+    def __init__(self, emb_dim: int) -> None:
+        super().__init__(aggr="add")
+        self.mlp = nn.Sequential(
+            nn.Linear(emb_dim, 2 * emb_dim), nn.BatchNorm1d(2 * emb_dim),
+            nn.ReLU(), nn.Linear(2 * emb_dim, emb_dim),
+        )
+        self.eps = nn.Parameter(torch.Tensor([0]))
+        self.bond_encoder = BondEncoder(emb_dim=emb_dim)
+
+    def forward(self, x, edge_index, edge_attr):
+        e = self.bond_encoder(edge_attr)
+        return self.mlp((1 + self.eps) * x + self.propagate(edge_index, x=x, edge_attr=e))
+
+    def message(self, x_j, edge_attr):
+        return F.relu(x_j + edge_attr)
+
+    def update(self, aggr_out):
+        return aggr_out
+
+
+class GNNComplete(nn.Module):
+    def __init__(self, num_layer: int = 5, emb_dim: int = 300, drop_ratio: float = 0.0) -> None:
+        super().__init__()
+        self.num_layer, self.drop_ratio = num_layer, drop_ratio
+        self.atom_encoder = AtomEncoder(emb_dim)
+        self.gnns = nn.ModuleList([GINConv(emb_dim) for _ in range(num_layer)])
+        self.batch_norms = nn.ModuleList([nn.BatchNorm1d(emb_dim) for _ in range(num_layer)])
+
+    def forward(self, x, edge_index, edge_attr):
+        h = self.atom_encoder(x)
+        for layer in range(self.num_layer):
+            h = self.gnns[layer](h, edge_index, edge_attr)
+            h = self.batch_norms[layer](h)
+            h = F.dropout(h if layer == self.num_layer - 1 else F.relu(h),
+                          self.drop_ratio, training=self.training)
+        return h
+
+
+def load_weights(model: nn.Module, ckpt_path: str) -> None:
+    sd = torch.load(ckpt_path, map_location="cpu")
+    msd = model.state_dict()
+    loaded, skipped, partial = 0, [], []
+    for k, v in sd.items():
+        if k not in msd:
+            continue
+        if msd[k].shape == v.shape:
+            msd[k] = v
+            loaded += 1
+        elif v.dim() == 2 and msd[k].shape[1] == v.shape[1] and msd[k].shape[0] > v.shape[0]:
+            msd[k][: v.shape[0]] = v          # OGB 类别数变化，前 N 行用预训练
+            partial.append(k)
+            loaded += 1
+        else:
+            skipped.append((k, tuple(v.shape), tuple(msd[k].shape)))
+    model.load_state_dict(msd)
+    print(f"已加载 {loaded}/{len(sd)} 个张量 | 部分加载: {partial} | 跳过: {skipped}")
+    assert not skipped, "有张量形状不兼容"
+
+
+def main(ckpt: str, csv_path: str, out_npy: str, col: str = "smiles", batch: int = 64) -> None:
+    model = GNNComplete().cuda().eval()
+    load_weights(model, ckpt)
+
+    smi = pd.read_csv(csv_path, low_memory=False)[col].astype(str).tolist()
+    uniq = list(dict.fromkeys(smi))
+    print(f"{len(smi)} 行, {len(uniq)} 唯一分子")
+
+    lut = {}
+    with torch.no_grad():
+        for i in range(0, len(uniq), batch):
+            datas = []
+            for s in uniq[i:i + batch]:
+                g = smiles2graph(s)
+                datas.append(Data(
+                    x=torch.tensor(g["node_feat"], dtype=torch.long),
+                    edge_index=torch.tensor(g["edge_index"], dtype=torch.long),
+                    edge_attr=torch.tensor(g["edge_feat"], dtype=torch.long),
+                ))
+            b = Batch.from_data_list(datas).cuda()
+            hg = global_mean_pool(model(b.x, b.edge_index, b.edge_attr), b.batch).cpu().numpy()
+            for j, s in enumerate(uniq[i:i + batch]):
+                lut[s] = hg[j]
+
+    E = np.stack([lut[s] for s in smi])
+    np.save(out_npy, E)
+    print(f"{out_npy} {E.shape}")
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser()
+    p.add_argument("--ckpt", required=True)
+    p.add_argument("--csv", required=True)
+    p.add_argument("--out", required=True)
+    p.add_argument("--col", default="smiles")
+    a = p.parse_args()
+    main(a.ckpt, a.csv, a.out, a.col)
diff --git a/scripts_run/encode_embeddings/encode_grover.py b/scripts_run/encode_embeddings/encode_grover.py
new file mode 100644
index 0000000..a00037d
--- /dev/null
+++ b/scripts_run/encode_embeddings/encode_grover.py
@@ -0,0 +1,57 @@
+"""GROVER fingerprint 离线编码 (3200d = atom 1600 + bond 1600)
+
+代码：https://github.com/tencent-ailab/grover ，权重 grover_base
+原项目基于 Python 3.6.8 / PyTorch 1.1，在现代环境需对 grover 源码打两处补丁：
+  1. grover/util/utils.py 的 torch.load 加 weights_only=False
+  2. build_model 前用 checkpoint 自带 args 补齐 current_args 缺失字段，
+     并为 dropout / features_only / ffn_num_layers 等推理期字段提供默认值
+
+三步流程：
+  # 1) 生成唯一 SMILES 输入
+  python encode_grover.py prepare --csv data/interim/internal.csv --out /tmp/grover_in.csv
+  # 2) 跑 GROVER fingerprint（在 grover 仓库目录下）
+  python main.py fingerprint --data_path /tmp/grover_in.csv \
+      --checkpoint_path grover_base.pt --fingerprint_source both --output /tmp/grover_fp.npz
+  # 3) 按 csv 行顺序对齐存 npy
+  python encode_grover.py align --fp /tmp/grover_fp.npz --input /tmp/grover_in.csv \
+      --target data/interim/internal.csv --out data/interim/grover_emb.npy
+"""
+import argparse
+import numpy as np
+import pandas as pd
+
+
+def prepare(csv_path: str, out_path: str, col: str = "smiles") -> None:
+    smi = pd.read_csv(csv_path, low_memory=False)[col].astype(str).tolist()
+    uniq = list(dict.fromkeys(smi))
+    pd.DataFrame({"smiles": uniq, "label": [0] * len(uniq)}).to_csv(out_path, index=False)
+    print(f"{len(smi)} 行 -> {len(uniq)} 唯一分子 -> {out_path}")
+
+
+def align(fp_npz: str, input_csv: str, target_csv: str, out_npy: str, col: str = "smiles") -> None:
+    fps = np.load(fp_npz)["fps"]
+    uniq = pd.read_csv(input_csv)["smiles"].astype(str).tolist()
+    assert len(uniq) == fps.shape[0], f"{len(uniq)} vs {fps.shape[0]}"
+    lut = {s: fps[i] for i, s in enumerate(uniq)}
+    rows = pd.read_csv(target_csv, low_memory=False)[col].astype(str).tolist()
+    E = np.stack([lut[s] for s in rows])
+    np.save(out_npy, E)
+    print(f"{out_npy} {E.shape}")
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser()
+    sub = p.add_subparsers(dest="cmd", required=True)
+    a = sub.add_parser("prepare")
+    a.add_argument("--csv", required=True)
+    a.add_argument("--out", required=True)
+    b = sub.add_parser("align")
+    b.add_argument("--fp", required=True)
+    b.add_argument("--input", required=True)
+    b.add_argument("--target", required=True)
+    b.add_argument("--out", required=True)
+    args = p.parse_args()
+    if args.cmd == "prepare":
+        prepare(args.csv, args.out)
+    else:
+        align(args.fp, args.input, args.target, args.out)
diff --git a/scripts_run/encode_embeddings/encode_molformer.py b/scripts_run/encode_embeddings/encode_molformer.py
new file mode 100644
index 0000000..793249c
--- /dev/null
+++ b/scripts_run/encode_embeddings/encode_molformer.py
@@ -0,0 +1,51 @@
+"""MolFormer-XL 离线 embedding (768d)
+
+依赖：transformers>=4.57（需 transformers.masking_utils）。若与项目 pin 的
+4.45 冲突，装到独立目录后用 sys.path 隔离：
+    pip install --target=/tmp/mf_libs "transformers>=4.57" --no-deps
+    pip install --target=/tmp/mf_libs tokenizers>=0.22,<=0.23 --no-deps
+    # 本脚本开头 sys.path.insert(0, '/tmp/mf_libs')
+
+用法：
+    python encode_molformer.py --csv data/interim/internal.csv \
+                               --out data/interim/molformer_emb.npy
+"""
+import argparse
+import numpy as np
+import pandas as pd
+import torch
+
+MODEL = "ibm/MoLFormer-XL-both-10pct"
+
+
+def main(csv_path: str, out_npy: str, col: str = "smiles", batch: int = 32) -> None:
+    from transformers import AutoModel, AutoTokenizer
+
+    smiles = pd.read_csv(csv_path, low_memory=False)[col].astype(str).tolist()
+    print(f"{len(smiles)} 行, {len(set(smiles))} 唯一分子")
+
+    tok = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=True)
+    model = AutoModel.from_pretrained(
+        MODEL, trust_remote_code=True, deterministic_eval=True
+    ).cuda().eval()
+
+    embs = []
+    with torch.no_grad():
+        for i in range(0, len(smiles), batch):
+            b = tok(smiles[i:i + batch], padding=True, truncation=True,
+                    max_length=512, return_tensors="pt")
+            b = {k: v.cuda() for k, v in b.items()}
+            embs.append(model(**b).pooler_output.cpu().numpy())
+
+    E = np.vstack(embs)
+    np.save(out_npy, E)
+    print(f"{out_npy} {E.shape}")
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser()
+    p.add_argument("--csv", required=True)
+    p.add_argument("--out", required=True)
+    p.add_argument("--col", default="smiles")
+    a = p.parse_args()
+    main(a.csv, a.out, a.col)