diff --git a/.gitignore b/.gitignore index 308b463..2207f21 100644 --- a/.gitignore +++ b/.gitignore @@ -202,3 +202,11 @@ models/**/*.sqlite3 models/biot5-plus-base/ data/processed/*.npz + +MolE_ckpt/ +MoleculeSTM_ckpt/ +MolE_repo/ +MoleculeSTM_repo/ + +data/processed/mole_representation.tsv.gz +data/processed/mole_input_smiles.csv diff --git a/lnp_ml/modeling/encoders/__init__.py b/lnp_ml/modeling/encoders/__init__.py index eb78d5c..8e4b7ee 100644 --- a/lnp_ml/modeling/encoders/__init__.py +++ b/lnp_ml/modeling/encoders/__init__.py @@ -2,10 +2,14 @@ from lnp_ml.modeling.encoders.rdkit_encoder import CachedRDKitEncoder from lnp_ml.modeling.encoders.mpnn_encoder import CachedMPNNEncoder from lnp_ml.modeling.encoders.chemeleon_encoder import CheMeleonEmbeddingEncoder from lnp_ml.modeling.encoders.unimol_encoder import UniMolEmbeddingEncoder +from lnp_ml.modeling.encoders.moleculestm_encoder import MoleculeSTMEmbeddingEncoder +from lnp_ml.modeling.encoders.mole_encoder import MolEEmbeddingEncoder __all__ = [ "CachedRDKitEncoder", "CachedMPNNEncoder", "CheMeleonEmbeddingEncoder", "UniMolEmbeddingEncoder", + "MoleculeSTMEmbeddingEncoder", + "MolEEmbeddingEncoder", ] \ No newline at end of file diff --git a/lnp_ml/modeling/encoders/mole_encoder.py b/lnp_ml/modeling/encoders/mole_encoder.py new file mode 100644 index 0000000..4b81660 --- /dev/null +++ b/lnp_ml/modeling/encoders/mole_encoder.py @@ -0,0 +1,48 @@ +"""MolE 预计算表征的查表编码器""" +from pathlib import Path +from typing import Dict, List + +import numpy as np +import torch +import torch.nn as nn + + +class MolEEmbeddingEncoder(nn.Module): + """从预计算 .npz 缓存按 SMILES 查 MolE 表征，返回 {"mole": [B, D]}。""" + + def __init__(self, cache_path: str) -> None: + super().__init__() + self.cache_path = str(cache_path) + self._table: Dict[str, np.ndarray] = {} + self._embed_dim: int = 0 + self._load() + + def _load(self) -> None: + path = Path(self.cache_path) + if not path.exists(): + raise FileNotFoundError( + f"MolE 缓存不存在: {path}。请先运行 scripts/precompute_mole.py。" + ) + data = np.load(path, allow_pickle=True) + embeddings = np.asarray(data["embeddings"], dtype=np.float32) + if embeddings.ndim != 2: + raise ValueError(f"embeddings 应为 2D，实际 {embeddings.shape}") + self._embed_dim = int(embeddings.shape[1]) + self._table = {str(s): embeddings[i] for i, s in enumerate(data["smiles"])} + + def forward(self, smiles_list: List[str]) -> Dict[str, torch.Tensor]: + missing = [s for s in smiles_list if s not in self._table] + if missing: + raise KeyError( + f"{len(missing)} 个 SMILES 不在 MolE 缓存中，请重跑 precompute_mole.py。" + f"示例: {missing[:3]}" + ) + mat = np.stack([self._table[s] for s in smiles_list]) + return {"mole": torch.from_numpy(mat)} + + def clear_cache(self) -> None: + """查表器无临时缓存，仅为接口对齐。""" + + @property + def embed_dim(self) -> int: + return self._embed_dim \ No newline at end of file diff --git a/lnp_ml/modeling/encoders/moleculestm_encoder.py b/lnp_ml/modeling/encoders/moleculestm_encoder.py new file mode 100644 index 0000000..6625e95 --- /dev/null +++ b/lnp_ml/modeling/encoders/moleculestm_encoder.py @@ -0,0 +1,48 @@ +"""MoleculeSTM 预计算表征的查表编码器""" +from pathlib import Path +from typing import Dict, List + +import numpy as np +import torch +import torch.nn as nn + + +class MoleculeSTMEmbeddingEncoder(nn.Module): + """从预计算 .npz 缓存按 SMILES 查 MoleculeSTM 表征，返回 {"moleculestm": [B, D]}。""" + + def __init__(self, cache_path: str) -> None: + super().__init__() + self.cache_path = str(cache_path) + self._table: Dict[str, np.ndarray] = {} + self._embed_dim: int = 0 + self._load() + + def _load(self) -> None: + path = Path(self.cache_path) + if not path.exists(): + raise FileNotFoundError( + f"MoleculeSTM 缓存不存在: {path}。请先运行 scripts/precompute_moleculestm.py。" + ) + data = np.load(path, allow_pickle=True) + embeddings = np.asarray(data["embeddings"], dtype=np.float32) + if embeddings.ndim != 2: + raise ValueError(f"embeddings 应为 2D，实际 {embeddings.shape}") + self._embed_dim = int(embeddings.shape[1]) + self._table = {str(s): embeddings[i] for i, s in enumerate(data["smiles"])} + + def forward(self, smiles_list: List[str]) -> Dict[str, torch.Tensor]: + missing = [s for s in smiles_list if s not in self._table] + if missing: + raise KeyError( + f"{len(missing)} 个 SMILES 不在 MoleculeSTM 缓存中，请重跑 precompute_moleculestm.py。" + f"示例: {missing[:3]}" + ) + mat = np.stack([self._table[s] for s in smiles_list]) + return {"moleculestm": torch.from_numpy(mat)} + + def clear_cache(self) -> None: + """查表器无临时缓存，仅为接口对齐。""" + + @property + def embed_dim(self) -> int: + return self._embed_dim \ No newline at end of file diff --git a/lnp_ml/modeling/models.py b/lnp_ml/modeling/models.py index 87e6643..395f083 100644 --- a/lnp_ml/modeling/models.py +++ b/lnp_ml/modeling/models.py @@ -9,6 +9,8 @@ from lnp_ml.modeling.encoders import ( CachedMPNNEncoder, CheMeleonEmbeddingEncoder, UniMolEmbeddingEncoder, + MoleculeSTMEmbeddingEncoder, + MolEEmbeddingEncoder, ) from lnp_ml.modeling.layers import ( TokenProjector, @@ -100,6 +102,10 @@ class LNPModel(nn.Module): chemeleon_cache_path: Optional[str] = None, # UniMol encoder unimol_cache_path: Optional[str] = None, + # MoleculeSTM encoder + moleculestm_cache_path: Optional[str] = None, + # MolE encoder + mole_cache_path: Optional[str] = None, # 输入维度配置 input_dims: Optional[Dict[str, int]] = None, # ============ MoE 相关 ============ @@ -132,6 +138,8 @@ class LNPModel(nn.Module): self.use_mpnn = mpnn_checkpoint is not None or mpnn_ensemble_paths is not None self.use_chemeleon = chemeleon_cache_path is not None self.use_unimol = unimol_cache_path is not None + self.use_moleculestm = moleculestm_cache_path is not None + self.use_mole = mole_cache_path is not None # ============ Encoders ============ self.rdkit_encoder = CachedRDKitEncoder() @@ -156,6 +164,18 @@ class LNPModel(nn.Module): else: self.unimol_encoder = None + if self.use_moleculestm: + self.moleculestm_encoder = MoleculeSTMEmbeddingEncoder(cache_path=moleculestm_cache_path) + self.input_dims = {**self.input_dims, "moleculestm": self.moleculestm_encoder.embed_dim} + else: + self.moleculestm_encoder = None + + if self.use_mole: + self.mole_encoder = MolEEmbeddingEncoder(cache_path=mole_cache_path) + self.input_dims = {**self.input_dims, "mole": self.mole_encoder.embed_dim} + else: + self.mole_encoder = None + # ============ Token Projector ============ proj_input_dims = {k: v for k, v in self.input_dims.items()} if not self.use_mpnn: @@ -174,6 +194,10 @@ class LNPModel(nn.Module): chem_keys.append("chemeleon") if self.use_unimol: chem_keys.append("unimol") + if self.use_moleculestm: + chem_keys.append("moleculestm") + if self.use_mole: + chem_keys.append("mole") chem_keys += ["morgan", "maccs", "desc"] self.chem_keys = chem_keys self.tab_keys = TAB_KEYS @@ -268,6 +292,10 @@ class LNPModel(nn.Module): all_features["chemeleon"] = self.chemeleon_encoder(smiles)["chemeleon"].to(device) if self.use_unimol: all_features["unimol"] = self.unimol_encoder(smiles)["unimol"].to(device) + if self.use_moleculestm: + all_features["moleculestm"] = self.moleculestm_encoder(smiles)["moleculestm"].to(device) + if self.use_mole: + all_features["mole"] = self.mole_encoder(smiles)["mole"].to(device) all_features["morgan"] = rdkit_features["morgan"].to(device) all_features["maccs"] = rdkit_features["maccs"].to(device) all_features["desc"] = rdkit_features["desc"].to(device) @@ -493,6 +521,8 @@ class LNPModelWithoutMPNN(LNPModel): input_dims: Optional[Dict[str, int]] = None, chemeleon_cache_path: Optional[str] = None, unimol_cache_path: Optional[str] = None, + moleculestm_cache_path: Optional[str] = None, + mole_cache_path: Optional[str] = None, # ============ MoE 相关 ============ use_moe: bool = False, moe_n_experts: int = 4, @@ -531,6 +561,8 @@ class LNPModelWithoutMPNN(LNPModel): mpnn_ensemble_paths=None, chemeleon_cache_path=chemeleon_cache_path, unimol_cache_path=unimol_cache_path, + moleculestm_cache_path=moleculestm_cache_path, + mole_cache_path=mole_cache_path, input_dims=dims, reg_bypass=reg_bypass, use_moe=use_moe, diff --git a/lnp_ml/modeling/nested_cv_optuna.py b/lnp_ml/modeling/nested_cv_optuna.py index 6b0a72c..e2ab035 100644 --- a/lnp_ml/modeling/nested_cv_optuna.py +++ b/lnp_ml/modeling/nested_cv_optuna.py @@ -197,6 +197,8 @@ def create_model( mpnn_device: str = "cpu", chemeleon_cache: Optional[str] = None, unimol_cache: Optional[str] = None, + moleculestm_cache: Optional[str] = None, + mole_cache: Optional[str] = None, set_transformer_block: str = "sab", # MoE use_moe: bool = False, @@ -222,6 +224,8 @@ def create_model( retr_feature_dim=retr_feature_dim, chemeleon_cache_path=chemeleon_cache, unimol_cache_path=unimol_cache, + moleculestm_cache_path=moleculestm_cache, + mole_cache_path=mole_cache, **(llm_kwargs or {}), ) @@ -424,6 +428,8 @@ def run_inner_optuna( use_mpnn: bool = False, chemeleon_cache: Optional[str] = None, unimol_cache: Optional[str] = None, + moleculestm_cache: Optional[str] = None, + mole_cache: Optional[str] = None, seed: int = 42, study_path: Optional[Path] = None, pretrain_state_dict: Optional[Dict] = None, @@ -544,6 +550,8 @@ def run_inner_optuna( mpnn_device=device.type, chemeleon_cache=chemeleon_cache, unimol_cache=unimol_cache, + moleculestm_cache=moleculestm_cache, + mole_cache=mole_cache, use_moe=use_moe, moe_n_experts=moe_ne_t, moe_top_k=moe_tk_t, @@ -641,6 +649,8 @@ def _run_single_outer_fold( use_mpnn: bool, chemeleon_cache: Optional[str], unimol_cache: Optional[str], + moleculestm_cache: Optional[str], + mole_cache: Optional[str], seed: int, pretrain_state_dict: Optional[Dict], pretrain_config: Optional[Dict], @@ -744,6 +754,8 @@ def _run_single_outer_fold( use_mpnn=use_mpnn, chemeleon_cache=chemeleon_cache, unimol_cache=unimol_cache, + moleculestm_cache=moleculestm_cache, + mole_cache=mole_cache, seed=seed + outer_fold, study_path=study_path, pretrain_state_dict=pretrain_state_dict, @@ -803,6 +815,8 @@ def _run_single_outer_fold( mpnn_device=device.type, chemeleon_cache=chemeleon_cache, unimol_cache=unimol_cache, + moleculestm_cache=moleculestm_cache, + mole_cache=mole_cache, use_moe=use_moe, moe_n_experts=best_params.get("moe_n_experts", moe_n_experts), moe_top_k=best_params.get("moe_top_k", moe_top_k), @@ -871,6 +885,10 @@ def _run_single_outer_fold( "chemeleon_cache": chemeleon_cache, "use_unimol": unimol_cache is not None, "unimol_cache": unimol_cache, + "use_moleculestm": moleculestm_cache is not None, + "moleculestm_cache": moleculestm_cache, + "use_mole": mole_cache is not None, + "mole_cache": mole_cache, "use_moe": use_moe, "moe_n_experts": moe_n_experts, "moe_top_k": moe_top_k, @@ -943,6 +961,10 @@ def main( chemeleon_cache: str = "data/processed/chemeleon_embeddings.npz", use_unimol: bool = False, unimol_cache: str = "data/processed/unimol_embeddings.npz", + use_moleculestm: bool = False, + moleculestm_cache: str = "data/processed/moleculestm_embeddings.npz", + use_mole: bool = False, + mole_cache: str = "data/processed/mole_embeddings.npz", n_repeats: int = 1, repeat_seed_step: int = 1000, # MoE（消融开关） @@ -1075,6 +1097,8 @@ def main( use_mpnn=use_mpnn, chemeleon_cache=(chemeleon_cache if use_chemeleon else None), unimol_cache=(unimol_cache if use_unimol else None), + moleculestm_cache=(moleculestm_cache if use_moleculestm else None), + mole_cache=(mole_cache if use_mole else None), seed=seed, pretrain_state_dict=pretrain_state_dict, pretrain_config=pretrain_config, diff --git a/models/abl_full/rf/seed42/outer_fold_0/test_metrics.json b/models/abl_full/rf/seed42/outer_fold_0/test_metrics.json new file mode 100644 index 0000000..2bd2ce7 --- /dev/null +++ b/models/abl_full/rf/seed42/outer_fold_0/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 83, + "mse": 1.368858521000182, + "rmse": 1.1699822737974204, + "mae": 0.48265016079386586, + "r2": 0.16222489685168928 + }, + "delivery": { + "n_samples": 58, + "mse": 0.4106486248647463, + "rmse": 0.6408187145088275, + "mae": 0.4092221012826776, + "r2": 0.4780408138229023 + }, + "toxic": { + "n_samples": 58, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.8095238095238095, + "precision": 0.7797101449275362, + "recall": 0.7063435495367071, + "f1": 0.7279352226720648 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.75, + "precision": 0.6912280701754385, + "recall": 0.6471428571428571, + "f1": 0.6631611379274931 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.16366098804686893, + "js_divergence": 0.0359968761663274 + } +} \ No newline at end of file diff --git a/models/abl_full/rf/seed42/outer_fold_1/test_metrics.json b/models/abl_full/rf/seed42/outer_fold_1/test_metrics.json new file mode 100644 index 0000000..613583a --- /dev/null +++ b/models/abl_full/rf/seed42/outer_fold_1/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.30493822479834903, + "rmse": 0.5522121193874225, + "mae": 0.34891908283267253, + "r2": 0.0688270762246912 + }, + "delivery": { + "n_samples": 61, + "mse": 0.7776284206466513, + "rmse": 0.8818324220886026, + "mae": 0.519796750021438, + "r2": 0.4113913613868899 + }, + "toxic": { + "n_samples": 61, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7619047619047619, + "precision": 0.6711111111111111, + "recall": 0.5873015873015873, + "f1": 0.5942028985507246 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6062341913405743, + "recall": 0.6071785357499643, + "f1": 0.6041666666666666 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.1177531067102451, + "js_divergence": 0.027774748827162937 + } +} \ No newline at end of file diff --git a/models/abl_full/rf/seed42/outer_fold_2/test_metrics.json b/models/abl_full/rf/seed42/outer_fold_2/test_metrics.json new file mode 100644 index 0000000..c33e2c9 --- /dev/null +++ b/models/abl_full/rf/seed42/outer_fold_2/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.3089202253953697, + "rmse": 0.5558059242175902, + "mae": 0.3994691472061615, + "r2": 0.3898880719888205 + }, + "delivery": { + "n_samples": 60, + "mse": 0.6215887874238653, + "rmse": 0.7884090229213928, + "mae": 0.4654098044336172, + "r2": 0.17255759426449502 + }, + "toxic": { + "n_samples": 61, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7261904761904762, + "precision": 0.6428571428571428, + "recall": 0.6385630498533724, + "f1": 0.6405581395348837 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6547619047619048, + "precision": 0.5676343636869953, + "recall": 0.5378151260504201, + "f1": 0.5491025070464323 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.11869463263891707, + "js_divergence": 0.02929511833384903 + } +} \ No newline at end of file diff --git a/models/abl_full/rf/seed42/outer_fold_3/test_metrics.json b/models/abl_full/rf/seed42/outer_fold_3/test_metrics.json new file mode 100644 index 0000000..ff98c84 --- /dev/null +++ b/models/abl_full/rf/seed42/outer_fold_3/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 83, + "mse": 1.6827545971608378, + "rmse": 1.2972103133882484, + "mae": 0.5269114175759023, + "r2": 0.048221123148072476 + }, + "delivery": { + "n_samples": 59, + "mse": 0.8527625246024667, + "rmse": 0.9234514197306032, + "mae": 0.5525073854104011, + "r2": 0.1710134699931437 + }, + "toxic": { + "n_samples": 60, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.75, + "precision": 0.6676909569798068, + "recall": 0.6400293255131965, + "f1": 0.6493738819320214 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6309523809523809, + "precision": 0.5132964178288214, + "recall": 0.49489693313222727, + "f1": 0.4975934917795383 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.1458100103551393, + "js_divergence": 0.034398655034422854 + } +} \ No newline at end of file diff --git a/models/abl_full/rf/seed42/outer_fold_4/test_metrics.json b/models/abl_full/rf/seed42/outer_fold_4/test_metrics.json new file mode 100644 index 0000000..bcec505 --- /dev/null +++ b/models/abl_full/rf/seed42/outer_fold_4/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.539048457673242, + "rmse": 0.7341991948192548, + "mae": 0.41399459761950447, + "r2": 0.27868624280784415 + }, + "delivery": { + "n_samples": 58, + "mse": 0.8685551446715326, + "rmse": 0.9319630597140278, + "mae": 0.48785093074168856, + "r2": 0.16012146519578374 + }, + "toxic": { + "n_samples": 59, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7380952380952381, + "precision": 0.6515151515151515, + "recall": 0.6319648093841642, + "f1": 0.6390625 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6048840048840048, + "recall": 0.6175213675213675, + "f1": 0.6107062011796588 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.12253709805850461, + "js_divergence": 0.03003058068886421 + } +} \ No newline at end of file diff --git a/models/abl_full/rf/seed42/summary.json b/models/abl_full/rf/seed42/summary.json new file mode 100644 index 0000000..546d1b3 --- /dev/null +++ b/models/abl_full/rf/seed42/summary.json @@ -0,0 +1,287 @@ +{ + "fold_results": [ + { + "fold": 0, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.368858521000182, + "rmse": 1.1699822737974204, + "mae": 0.48265016079386586, + "r2": 0.16222489685168928 + }, + "delivery": { + "n_samples": 58, + "mse": 0.4106486248647463, + "rmse": 0.6408187145088275, + "mae": 0.4092221012826776, + "r2": 0.4780408138229023 + }, + "toxic": { + "n_samples": 58, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.8095238095238095, + "precision": 0.7797101449275362, + "recall": 0.7063435495367071, + "f1": 0.7279352226720648 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.75, + "precision": 0.6912280701754385, + "recall": 0.6471428571428571, + "f1": 0.6631611379274931 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.16366098804686893, + "js_divergence": 0.0359968761663274 + } + } + }, + { + "fold": 1, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.30493822479834903, + "rmse": 0.5522121193874225, + "mae": 0.34891908283267253, + "r2": 0.0688270762246912 + }, + "delivery": { + "n_samples": 61, + "mse": 0.7776284206466513, + "rmse": 0.8818324220886026, + "mae": 0.519796750021438, + "r2": 0.4113913613868899 + }, + "toxic": { + "n_samples": 61, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7619047619047619, + "precision": 0.6711111111111111, + "recall": 0.5873015873015873, + "f1": 0.5942028985507246 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6062341913405743, + "recall": 0.6071785357499643, + "f1": 0.6041666666666666 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.1177531067102451, + "js_divergence": 0.027774748827162937 + } + } + }, + { + "fold": 2, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.3089202253953697, + "rmse": 0.5558059242175902, + "mae": 0.3994691472061615, + "r2": 0.3898880719888205 + }, + "delivery": { + "n_samples": 60, + "mse": 0.6215887874238653, + "rmse": 0.7884090229213928, + "mae": 0.4654098044336172, + "r2": 0.17255759426449502 + }, + "toxic": { + "n_samples": 61, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7261904761904762, + "precision": 0.6428571428571428, + "recall": 0.6385630498533724, + "f1": 0.6405581395348837 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6547619047619048, + "precision": 0.5676343636869953, + "recall": 0.5378151260504201, + "f1": 0.5491025070464323 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.11869463263891707, + "js_divergence": 0.02929511833384903 + } + } + }, + { + "fold": 3, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.6827545971608378, + "rmse": 1.2972103133882484, + "mae": 0.5269114175759023, + "r2": 0.048221123148072476 + }, + "delivery": { + "n_samples": 59, + "mse": 0.8527625246024667, + "rmse": 0.9234514197306032, + "mae": 0.5525073854104011, + "r2": 0.1710134699931437 + }, + "toxic": { + "n_samples": 60, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.75, + "precision": 0.6676909569798068, + "recall": 0.6400293255131965, + "f1": 0.6493738819320214 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6309523809523809, + "precision": 0.5132964178288214, + "recall": 0.49489693313222727, + "f1": 0.4975934917795383 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.1458100103551393, + "js_divergence": 0.034398655034422854 + } + } + }, + { + "fold": 4, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.539048457673242, + "rmse": 0.7341991948192548, + "mae": 0.41399459761950447, + "r2": 0.27868624280784415 + }, + "delivery": { + "n_samples": 58, + "mse": 0.8685551446715326, + "rmse": 0.9319630597140278, + "mae": 0.48785093074168856, + "r2": 0.16012146519578374 + }, + "toxic": { + "n_samples": 59, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7380952380952381, + "precision": 0.6515151515151515, + "recall": 0.6319648093841642, + "f1": 0.6390625 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6048840048840048, + "recall": 0.6175213675213675, + "f1": 0.6107062011796588 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.12253709805850461, + "js_divergence": 0.03003058068886421 + } + } + } + ], + "summary_stats": { + "size": { + "mse_mean": 0.840904005205596, + "mse_std": 0.5742527703456151, + "rmse_mean": 0.8618819651219873, + "rmse_std": 0.3131508955808008, + "mae_mean": 0.4343888812056213, + "mae_std": 0.06297614136064734, + "r2_mean": 0.1895694822042235, + "r2_std": 0.12913183030644884 + }, + "delivery": { + "mse_mean": 0.7062367004418524, + "mse_std": 0.17172737219125497, + "rmse_mean": 0.8332949277926908, + "rmse_std": 0.1088864718724354, + "mae_mean": 0.4869573943779645, + "mae_std": 0.048750584479951106, + "r2_mean": 0.27862494093264295, + "r2_std": 0.13730782732933916 + }, + "toxic": { + "accuracy_mean": 1.0, + "accuracy_std": 0.0, + "precision_mean": 1.0, + "precision_std": 0.0, + "recall_mean": 1.0, + "recall_std": 0.0, + "f1_mean": 1.0, + "f1_std": 0.0 + }, + "pdi": { + "accuracy_mean": 0.7571428571428571, + "accuracy_std": 0.02876915707998708, + "precision_mean": 0.6825769014781496, + "precision_std": 0.04965967017816626, + "recall_mean": 0.6408404643178055, + "recall_std": 0.03805702915009694, + "f1_mean": 0.6502265285379389, + "f1_std": 0.04335003701571735 + }, + "ee": { + "accuracy_mean": 0.6785714285714286, + "accuracy_std": 0.03984095364447979, + "precision_mean": 0.5966554095831669, + "precision_std": 0.05812812328518918, + "recall_mean": 0.5809109639193673, + "recall_std": 0.055986847376187095, + "f1_mean": 0.5849460009199577, + "f1_std": 0.056684824199185184 + }, + "biodist": { + "kl_divergence_mean": 0.13369116716193502, + "kl_divergence_std": 0.01815702398766046, + "js_divergence_mean": 0.031499195810125294, + "js_divergence_std": 0.0031471347478191324 + } + } +} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_0/best_params.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_0/best_params.json new file mode 100644 index 0000000..fe857df --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_0/best_params.json @@ -0,0 +1,16 @@ +{ + "dropout": 0.4108948503325973, + "lr": 0.0007340472227986773, + "weight_decay": 0.0007067135047679004, + "backbone_lr_ratio": 0.7321173243252591, + "moe_n_experts": 2, + "moe_top_k": 2, + "moe_expert_hidden_mult": 1, + "llm_lora_r": 32, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_0/epoch_mean.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_0/epoch_mean.json new file mode 100644 index 0000000..82d35d4 --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_0/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 14} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_0/history.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_0/history.json new file mode 100644 index 0000000..aa03db5 --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_0/history.json @@ -0,0 +1,328 @@ +{ + "train": [ + { + "loss": 5.468005173915142, + "loss_size": 0.8984898155605471, + "loss_pdi": 0.6840105072872059, + "loss_ee": 1.081262314641798, + "loss_delivery": 1.0520821031485055, + "loss_biodist": 1.1721819816408932, + "loss_toxic": 0.5795586423293965, + "loss_moe_lb": 1.9999999871125091 + }, + { + "loss": 4.469357091027337, + "loss_size": 0.8436893965746906, + "loss_pdi": 0.6519882775641777, + "loss_ee": 0.9837553597785331, + "loss_delivery": 1.0650141696269448, + "loss_biodist": 0.6095170338411589, + "loss_toxic": 0.3277856332526819, + "loss_moe_lb": 1.9999999838906366 + }, + { + "loss": 3.816732780353443, + "loss_size": 0.7817897897314381, + "loss_pdi": 0.5992057540932217, + "loss_ee": 0.9323523914491808, + "loss_delivery": 0.9979917161368035, + "loss_biodist": 0.3502167326373023, + "loss_toxic": 0.1728987922338215, + "loss_moe_lb": 1.9999999935562547 + }, + { + "loss": 3.765748839120607, + "loss_size": 0.8649615081580909, + "loss_pdi": 0.5580552405602223, + "loss_ee": 0.9576937169642061, + "loss_delivery": 0.9454193354861157, + "loss_biodist": 0.33004482090473175, + "loss_toxic": 0.14383074739787452, + "loss_moe_lb": 1.9999999871125091 + }, + { + "loss": 3.7829968284916236, + "loss_size": 0.8822104828180494, + "loss_pdi": 0.5236790458092818, + "loss_ee": 0.935156566065711, + "loss_delivery": 0.9641252619189185, + "loss_biodist": 0.2763468524491465, + "loss_toxic": 0.2391656021346854, + "loss_moe_lb": 1.9999999967781272 + }, + { + "loss": 3.4474196756208264, + "loss_size": 0.7625373472031709, + "loss_pdi": 0.5673897685231389, + "loss_ee": 0.869621529772475, + "loss_delivery": 0.9728898971870139, + "loss_biodist": 0.19711698450752208, + "loss_toxic": 0.12803069363124794, + "loss_moe_lb": 1.9999999806687638 + }, + { + "loss": 3.3459737590841345, + "loss_size": 0.8519144344168741, + "loss_pdi": 0.5011695462304193, + "loss_ee": 0.8461442412556829, + "loss_delivery": 0.9089953640626894, + "loss_biodist": 0.20563854800688253, + "loss_toxic": 0.09120985464766351, + "loss_moe_lb": 1.9999999871125091 + }, + { + "loss": 3.2159800658354887, + "loss_size": 0.785401169512723, + "loss_pdi": 0.5035344394477638, + "loss_ee": 0.8079836134975021, + "loss_delivery": 0.8549810738579647, + "loss_biodist": 0.19876324878753843, + "loss_toxic": 0.12502085922116404, + "loss_moe_lb": 1.999999990334382 + }, + { + "loss": 3.1401129252201803, + "loss_size": 0.8098833548861581, + "loss_pdi": 0.5369876888958184, + "loss_ee": 0.7754076888432374, + "loss_delivery": 0.8422002055229166, + "loss_biodist": 0.18652975659918142, + "loss_toxic": 0.06130260335457687, + "loss_moe_lb": 1.999999990334382 + }, + { + "loss": 3.0456181732383936, + "loss_size": 0.7640989988639548, + "loss_pdi": 0.47404823049500183, + "loss_ee": 0.7642006817701701, + "loss_delivery": 0.8336286452170965, + "loss_biodist": 0.1764557785480409, + "loss_toxic": 0.10230353281745133, + "loss_moe_lb": 1.9999999967781272 + }, + { + "loss": 2.773993302035976, + "loss_size": 0.6818141031104166, + "loss_pdi": 0.47380794101470225, + "loss_ee": 0.7102676042028375, + "loss_delivery": 0.7278135028239843, + "loss_biodist": 0.17976967315818812, + "loss_toxic": 0.06622670966946217, + "loss_moe_lb": 1.9999999838906366 + }, + { + "loss": 2.83101592514966, + "loss_size": 0.6544620332282942, + "loss_pdi": 0.4825279181068008, + "loss_ee": 0.7465345295699867, + "loss_delivery": 0.81812868045794, + "loss_biodist": 0.16096079304210237, + "loss_toxic": 0.047566130741010106, + "loss_moe_lb": 1.999999977446891 + }, + { + "loss": 2.639647944553478, + "loss_size": 0.67716606082143, + "loss_pdi": 0.4616012508804734, + "loss_ee": 0.7141166007196581, + "loss_delivery": 0.6510741408612277, + "loss_biodist": 0.14901544113416929, + "loss_toxic": 0.05449561633272301, + "loss_moe_lb": 1.9999999935562547 + }, + { + "loss": 2.74934356921428, + "loss_size": 0.6282122729195131, + "loss_pdi": 0.4631339017603848, + "loss_ee": 0.6918598010733321, + "loss_delivery": 0.8057697197875461, + "loss_biodist": 0.16010575097154928, + "loss_toxic": 0.07083596092825001, + "loss_moe_lb": 1.9999999967781272 + } + ], + "val": [ + { + "loss": 4.029903936386108, + "loss_size": 0.4817116637481377, + "loss_pdi": 0.69562246799469, + "loss_ee": 1.0227925896644592, + "loss_delivery": 0.6827521236147731, + "loss_biodist": 0.8071430802345276, + "loss_toxic": 0.34477609395980835, + "loss_moe_lb": 1.9999999761581422, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.45454545454545453, + "acc_toxic": 1.0 + }, + { + "loss": 3.9466779708862303, + "loss_size": 0.48833362460136415, + "loss_pdi": 0.7379729390144348, + "loss_ee": 0.9316445291042328, + "loss_delivery": 0.6856113743036986, + "loss_biodist": 0.8218623757362366, + "loss_toxic": 0.3024887479841709, + "loss_moe_lb": 2.0000000476837156, + "acc_pdi": 0.48484848484848486, + "acc_ee": 0.45454545454545453, + "acc_toxic": 0.9 + }, + { + "loss": 4.236728382110596, + "loss_size": 0.5088113136589527, + "loss_pdi": 0.9482179045677185, + "loss_ee": 0.8386058039963246, + "loss_delivery": 0.6349728442728519, + "loss_biodist": 0.5330403506755829, + "loss_toxic": 0.7939994555985322, + "loss_moe_lb": 2.000000023841858, + "acc_pdi": 0.5151515151515151, + "acc_ee": 0.45454545454545453, + "acc_toxic": 0.9 + }, + { + "loss": 3.9902405738830566, + "loss_size": 0.5039946608245373, + "loss_pdi": 0.9336968839168549, + "loss_ee": 0.798806632310152, + "loss_delivery": 0.6531890798360109, + "loss_biodist": 0.2753306046128273, + "loss_toxic": 0.8350482980720699, + "loss_moe_lb": 1.9999999761581422, + "acc_pdi": 0.5151515151515151, + "acc_ee": 0.3939393939393939, + "acc_toxic": 0.85 + }, + { + "loss": 4.208648014068603, + "loss_size": 0.43180498518049715, + "loss_pdi": 0.9745973706245422, + "loss_ee": 0.8454247560352087, + "loss_delivery": 0.8437085047364234, + "loss_biodist": 0.2725426733493805, + "loss_toxic": 0.8599944234316353, + "loss_moe_lb": 2.0000000476837156, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.42424242424242425, + "acc_toxic": 0.9 + }, + { + "loss": 3.284090757369995, + "loss_size": 0.40962614729069174, + "loss_pdi": 0.9049368262290954, + "loss_ee": 0.8435465201735497, + "loss_delivery": 0.6470494873967254, + "loss_biodist": 0.1634147047996521, + "loss_toxic": 0.3562523804110242, + "loss_moe_lb": 2.0, + "acc_pdi": 0.5454545454545454, + "acc_ee": 0.42424242424242425, + "acc_toxic": 0.9 + }, + { + "loss": 3.755344009399414, + "loss_size": 0.3185691962018609, + "loss_pdi": 1.055866003036499, + "loss_ee": 0.855142249725759, + "loss_delivery": 0.7583918676711618, + "loss_biodist": 0.2227014943957329, + "loss_toxic": 0.5842763775202912, + "loss_moe_lb": 2.0, + "acc_pdi": 0.6060606060606061, + "acc_ee": 0.3939393939393939, + "acc_toxic": 0.9 + }, + { + "loss": 2.964364767074585, + "loss_size": 0.35715504018007777, + "loss_pdi": 0.8166249096393585, + "loss_ee": 0.8160352975130081, + "loss_delivery": 0.6897998858476058, + "loss_biodist": 0.23408670574426652, + "loss_toxic": 0.11860285412403755, + "loss_moe_lb": 1.9999999523162841, + "acc_pdi": 0.6363636363636364, + "acc_ee": 0.48484848484848486, + "acc_toxic": 1.0 + }, + { + "loss": 2.9165560722351076, + "loss_size": 0.3242662086704513, + "loss_pdi": 0.8310463488101959, + "loss_ee": 0.8328023519366979, + "loss_delivery": 0.5303224935662001, + "loss_biodist": 0.1381889261305332, + "loss_toxic": 0.30898305508017077, + "loss_moe_lb": 1.9999999523162841, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.45454545454545453, + "acc_toxic": 0.9 + }, + { + "loss": 2.8056929111480713, + "loss_size": 0.26238062530755996, + "loss_pdi": 0.8931824445724488, + "loss_ee": 0.7837101235985756, + "loss_delivery": 0.560417365655303, + "loss_biodist": 0.1655781462788582, + "loss_toxic": 0.20284669845823372, + "loss_moe_lb": 2.0, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.5151515151515151, + "acc_toxic": 0.95 + }, + { + "loss": 3.173236274719238, + "loss_size": 0.26263659158721564, + "loss_pdi": 0.9280910611152648, + "loss_ee": 0.834754678979516, + "loss_delivery": 0.6336017560213805, + "loss_biodist": 0.1626102939248085, + "loss_toxic": 0.4021385707083027, + "loss_moe_lb": 2.0000000476837156, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.5151515151515151, + "acc_toxic": 0.9 + }, + { + "loss": 3.200787878036499, + "loss_size": 0.2361290529370308, + "loss_pdi": 0.9936746537685395, + "loss_ee": 0.8308493941091001, + "loss_delivery": 0.5805998370051384, + "loss_biodist": 0.1770829126238823, + "loss_toxic": 0.42960651555149526, + "loss_moe_lb": 2.0, + "acc_pdi": 0.6363636363636364, + "acc_ee": 0.5151515151515151, + "acc_toxic": 0.9 + }, + { + "loss": 2.8428207874298095, + "loss_size": 0.25985395684838297, + "loss_pdi": 0.9453944981098175, + "loss_ee": 0.8041019512340426, + "loss_delivery": 0.5626729521900415, + "loss_biodist": 0.1452166758477688, + "loss_toxic": 0.19761992460371403, + "loss_moe_lb": 1.9999999761581422, + "acc_pdi": 0.6363636363636364, + "acc_ee": 0.5151515151515151, + "acc_toxic": 0.95 + }, + { + "loss": 2.868666648864746, + "loss_size": 0.23738904893398285, + "loss_pdi": 0.9498687088489532, + "loss_ee": 0.7990141914226114, + "loss_delivery": 0.5398828148841858, + "loss_biodist": 0.1589307226240635, + "loss_toxic": 0.24729853977114544, + "loss_moe_lb": 1.9999999523162841, + "acc_pdi": 0.6363636363636364, + "acc_ee": 0.5151515151515151, + "acc_toxic": 0.9 + } + ] +} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_0/splits.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_0/splits.json new file mode 100644 index 0000000..cccb9d8 --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_0/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 52, 53, 54, 55, 56, 57, 58, 61, 62, 63, 64, 65, 66, 67, 68, 71, 72, 73, 75, 76, 77, 80, 81, 85, 86, 88, 89, 90, 92, 94, 95, 96, 98, 99, 100, 101, 103, 104, 105, 106, 107, 108, 109, 111, 113, 114, 115, 116, 117, 118, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 133, 134, 135, 136, 138, 139, 141, 144, 145, 146, 147, 148, 149, 150, 151, 153, 154, 155, 156, 157, 158, 159, 160, 163, 164, 167, 168, 169, 170, 171, 172, 175, 176, 178, 179, 180, 182, 183, 184, 185, 188, 189, 190, 191, 192, 193, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 206, 208, 210, 211, 212, 213, 216, 217, 218, 219, 220, 222, 224, 225, 227, 228, 229, 230, 231, 232, 233, 234, 236, 237, 239, 241, 242, 244, 245, 247, 248, 249, 250, 252, 253, 255, 256, 257, 258, 259, 260, 261, 262, 265, 267, 268, 269, 270, 271, 272, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 285, 286, 287, 288, 291, 292, 294, 296, 297, 298, 299, 300, 301, 303, 304, 306, 307, 308, 309, 310, 311, 312, 313, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 332, 333, 335, 336, 337, 338, 339, 340, 342, 343, 344, 345, 346, 347, 348, 349, 351, 352, 354, 355, 356, 357, 358, 359, 360, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 374, 375, 377, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 393, 395, 396, 397, 398, 399, 400, 401, 403, 404, 405, 406, 407, 408, 409, 411, 412, 413, 414, 415, 417, 418, 419], "outer_test_idx": [7, 16, 20, 29, 51, 59, 60, 69, 70, 74, 78, 79, 82, 83, 84, 87, 91, 93, 97, 102, 110, 112, 119, 120, 121, 132, 137, 140, 142, 143, 152, 161, 162, 165, 166, 173, 174, 177, 181, 186, 187, 194, 205, 207, 209, 214, 215, 221, 223, 226, 235, 238, 240, 243, 246, 251, 254, 263, 264, 266, 273, 284, 289, 290, 293, 295, 302, 305, 314, 315, 331, 334, 341, 350, 353, 361, 373, 376, 378, 392, 394, 402, 410, 416]} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_0/test_metrics.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_0/test_metrics.json new file mode 100644 index 0000000..18ff7fc --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_0/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 83, + "mse": 1.566305704940777, + "rmse": 1.2515213561664766, + "mae": 0.6106541865583823, + "r2": 0.04138235411462465 + }, + "delivery": { + "n_samples": 58, + "mse": 0.5779477061981523, + "rmse": 0.7602287196614926, + "mae": 0.6029246019777553, + "r2": 0.2653935897935683 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.75, + "precision": 0.7018518518518518, + "recall": 0.733071988595866, + "f1": 0.7105824446267432 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.75, + "precision": 0.7208073948311639, + "recall": 0.7614285714285715, + "f1": 0.7248345524207593 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9655172413793104, + "precision": 0.75, + "recall": 0.9821428571428572, + "f1": 0.8242424242424242 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.21251985213285735, + "js_divergence": 0.04386748213277705 + } +} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_1/best_params.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_1/best_params.json new file mode 100644 index 0000000..acdba87 --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_1/best_params.json @@ -0,0 +1,16 @@ +{ + "dropout": 0.24482591245144014, + "lr": 0.00042458936788614755, + "weight_decay": 0.00014902327669828248, + "backbone_lr_ratio": 0.24808069123277332, + "moe_n_experts": 2, + "moe_top_k": 1, + "moe_expert_hidden_mult": 1, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_1/epoch_mean.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_1/epoch_mean.json new file mode 100644 index 0000000..89e65d1 --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_1/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 7} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_1/history.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_1/history.json new file mode 100644 index 0000000..51a45d3 --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_1/history.json @@ -0,0 +1,167 @@ +{ + "train": [ + { + "loss": 5.244253074800646, + "loss_size": 0.9031378044872671, + "loss_pdi": 0.6649061876374323, + "loss_ee": 1.0588606805414766, + "loss_delivery": 0.9051075132714735, + "loss_biodist": 1.1088146937860024, + "loss_toxic": 0.6031557522915505, + "loss_moe_lb": 1.004185162686013 + }, + { + "loss": 4.754348709776595, + "loss_size": 1.0869059155921679, + "loss_pdi": 0.6177884171137938, + "loss_ee": 0.9738765117284414, + "loss_delivery": 0.9496394661632744, + "loss_biodist": 0.7282283765238684, + "loss_toxic": 0.40549099284249385, + "loss_moe_lb": 1.0071367444218815 + }, + { + "loss": 3.806969288233164, + "loss_size": 0.7796624081360327, + "loss_pdi": 0.585498982184642, + "loss_ee": 0.8680859554458309, + "loss_delivery": 0.8664211886053955, + "loss_biodist": 0.49102446235514974, + "loss_toxic": 0.2227959665092262, + "loss_moe_lb": 1.006230396193427 + }, + { + "loss": 3.892503210016199, + "loss_size": 1.1313844786302463, + "loss_pdi": 0.5464168086245254, + "loss_ee": 0.8315383767759478, + "loss_delivery": 0.7725983503702525, + "loss_biodist": 0.40765431120588974, + "loss_toxic": 0.21178772657908299, + "loss_moe_lb": 1.0067882360638798 + }, + { + "loss": 3.483587944829786, + "loss_size": 0.9415902084595448, + "loss_pdi": 0.5138166675696502, + "loss_ee": 0.7523788245948585, + "loss_delivery": 0.7893044344476752, + "loss_biodist": 0.3645425988612948, + "loss_toxic": 0.129310202004539, + "loss_moe_lb": 1.0053398512505196 + }, + { + "loss": 3.409756396267865, + "loss_size": 1.0240287120277818, + "loss_pdi": 0.5097397509458903, + "loss_ee": 0.7305249813440684, + "loss_delivery": 0.7399266489051484, + "loss_biodist": 0.3145633094214104, + "loss_toxic": 0.09821102107447144, + "loss_moe_lb": 1.0032536532427814 + }, + { + "loss": 3.2366123457212708, + "loss_size": 0.9939367072405042, + "loss_pdi": 0.479262328228435, + "loss_ee": 0.6680147180686126, + "loss_delivery": 0.7013498719882321, + "loss_biodist": 0.30082041407759125, + "loss_toxic": 0.09702983350065109, + "loss_moe_lb": 1.0036999228838328 + } + ], + "val": [ + { + "loss": 3.5980638027191163, + "loss_size": 0.5808463346969802, + "loss_pdi": 0.6537732005119323, + "loss_ee": 0.9681090712547302, + "loss_delivery": 0.20582003718009217, + "loss_biodist": 0.8013778448104858, + "loss_toxic": 0.38751593232154846, + "loss_moe_lb": 1.0249070644378662, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.696969696969697, + "acc_toxic": 1.0 + }, + { + "loss": 2.7783058881759644, + "loss_size": 0.5202976800501347, + "loss_pdi": 0.6863118171691894, + "loss_ee": 0.8179924249649048, + "loss_delivery": 0.26323547810316084, + "loss_biodist": 0.37435545921325686, + "loss_toxic": 0.1097041517496109, + "loss_moe_lb": 1.0022525668144227, + "acc_pdi": 0.5454545454545454, + "acc_ee": 0.7272727272727273, + "acc_toxic": 1.0 + }, + { + "loss": 2.558013939857483, + "loss_size": 0.49102085679769514, + "loss_pdi": 0.6516811728477478, + "loss_ee": 0.6840570062398911, + "loss_delivery": 0.2801114499568939, + "loss_biodist": 0.4324764788150787, + "loss_toxic": 0.01573317525908351, + "loss_moe_lb": 1.0377389669418335, + "acc_pdi": 0.5757575757575758, + "acc_ee": 0.7575757575757576, + "acc_toxic": 1.0 + }, + { + "loss": 2.438762164115906, + "loss_size": 0.41186533421278, + "loss_pdi": 0.6200830161571502, + "loss_ee": 0.6213899791240692, + "loss_delivery": 0.40568435192108154, + "loss_biodist": 0.36440178751945496, + "loss_toxic": 0.013790842704474926, + "loss_moe_lb": 1.0057307958602906, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.7575757575757576, + "acc_toxic": 1.0 + }, + { + "loss": 2.371489381790161, + "loss_size": 0.3980108886957169, + "loss_pdi": 0.6092007637023926, + "loss_ee": 0.6023105025291443, + "loss_delivery": 0.3798559784889221, + "loss_biodist": 0.37731829583644866, + "loss_toxic": 0.004294451186433434, + "loss_moe_lb": 1.0124784469604493, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.7575757575757576, + "acc_toxic": 1.0 + }, + { + "loss": 2.2917377233505247, + "loss_size": 0.40251858830451964, + "loss_pdi": 0.5736695826053619, + "loss_ee": 0.5871740281581879, + "loss_delivery": 0.37468419075012205, + "loss_biodist": 0.3452415645122528, + "loss_toxic": 0.006651339470408857, + "loss_moe_lb": 1.028975820541382, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.7878787878787878, + "acc_toxic": 1.0 + }, + { + "loss": 2.2100016355514525, + "loss_size": 0.38368124067783355, + "loss_pdi": 0.5757645547389985, + "loss_ee": 0.5918506503105163, + "loss_delivery": 0.33769371509552004, + "loss_biodist": 0.3128642350435257, + "loss_toxic": 0.005693616252392531, + "loss_moe_lb": 1.0255095720291139, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.7878787878787878, + "acc_toxic": 1.0 + } + ] +} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_1/splits.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_1/splits.json new file mode 100644 index 0000000..1773fe5 --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_1/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [0, 3, 5, 6, 7, 10, 11, 12, 13, 15, 16, 18, 19, 20, 21, 24, 26, 27, 29, 30, 31, 32, 33, 35, 36, 38, 39, 40, 42, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 65, 66, 67, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 97, 98, 99, 100, 101, 102, 103, 105, 106, 107, 108, 109, 110, 112, 114, 115, 118, 119, 120, 121, 123, 124, 125, 126, 128, 129, 130, 131, 132, 136, 137, 140, 141, 142, 143, 144, 145, 146, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 161, 162, 164, 165, 166, 167, 168, 169, 171, 172, 173, 174, 175, 176, 177, 178, 179, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 194, 195, 196, 197, 198, 199, 201, 202, 204, 205, 206, 207, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 225, 226, 228, 229, 231, 232, 233, 234, 235, 236, 237, 238, 240, 242, 243, 244, 246, 247, 248, 250, 251, 252, 254, 255, 256, 258, 259, 260, 261, 262, 263, 264, 266, 268, 269, 270, 271, 273, 274, 275, 276, 277, 278, 279, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 295, 296, 297, 300, 301, 302, 303, 304, 305, 306, 307, 308, 310, 311, 313, 314, 315, 316, 317, 318, 319, 321, 322, 323, 324, 325, 327, 328, 329, 330, 331, 333, 334, 335, 337, 338, 339, 340, 341, 342, 343, 344, 345, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 361, 362, 364, 365, 366, 367, 368, 369, 373, 374, 375, 376, 378, 379, 380, 381, 382, 384, 387, 388, 389, 391, 392, 393, 394, 396, 397, 398, 399, 402, 403, 404, 405, 406, 408, 409, 410, 411, 412, 413, 414, 416, 417, 419], "outer_test_idx": [1, 2, 4, 8, 9, 14, 17, 22, 23, 25, 28, 34, 37, 41, 43, 45, 63, 64, 68, 85, 96, 104, 111, 113, 116, 117, 122, 127, 133, 134, 135, 138, 139, 147, 160, 163, 170, 180, 191, 192, 193, 200, 203, 208, 224, 227, 230, 239, 241, 245, 249, 253, 257, 265, 267, 272, 280, 294, 298, 299, 309, 312, 320, 326, 332, 336, 346, 347, 360, 363, 370, 371, 372, 377, 383, 385, 386, 390, 395, 400, 401, 407, 415, 418]} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_1/test_metrics.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_1/test_metrics.json new file mode 100644 index 0000000..c6a79fe --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_1/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.35709959292610643, + "rmse": 0.597578106130158, + "mae": 0.46698603426505414, + "r2": -0.09045519979354744 + }, + "delivery": { + "n_samples": 61, + "mse": 1.0490149027835551, + "rmse": 1.0242142855787333, + "mae": 0.6541144933062987, + "r2": 0.20597134095659064 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.5974801061007957, + "recall": 0.6111111111111112, + "f1": 0.6011957095129241 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5833333333333334, + "precision": 0.5340802987861811, + "recall": 0.5444079729794016, + "f1": 0.5238095238095238 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9508196721311475, + "precision": 0.75, + "recall": 0.9741379310344828, + "f1": 0.8200589970501475 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.35185404638391565, + "js_divergence": 0.09285944999940988 + } +} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_2/best_params.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_2/best_params.json new file mode 100644 index 0000000..1291ad2 --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_2/best_params.json @@ -0,0 +1,16 @@ +{ + "dropout": 0.330147494187667, + "lr": 0.0007391626870428583, + "weight_decay": 0.08248214185502709, + "backbone_lr_ratio": 0.3321485864522685, + "moe_n_experts": 4, + "moe_top_k": 1, + "moe_expert_hidden_mult": 1, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_2/epoch_mean.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_2/epoch_mean.json new file mode 100644 index 0000000..17fb17e --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_2/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 10} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_2/history.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_2/history.json new file mode 100644 index 0000000..f672a11 --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_2/history.json @@ -0,0 +1,236 @@ +{ + "train": [ + { + "loss": 5.200881294302039, + "loss_size": 0.8548808800610336, + "loss_pdi": 0.6923638227823619, + "loss_ee": 1.0423318083221849, + "loss_delivery": 0.9569440728707893, + "loss_biodist": 1.1431677357570544, + "loss_toxic": 0.5179293743661932, + "loss_moe_lb": 1.0149018087902584 + }, + { + "loss": 4.194867875124957, + "loss_size": 0.8379024593411265, + "loss_pdi": 0.6478910454221674, + "loss_ee": 0.9205394983291626, + "loss_delivery": 0.9244052073037302, + "loss_biodist": 0.5440030613461057, + "loss_toxic": 0.33539649904579727, + "loss_moe_lb": 1.0106659370499689 + }, + { + "loss": 3.6746369117015116, + "loss_size": 0.807164061713863, + "loss_pdi": 0.6030087317969348, + "loss_ee": 0.880505075325837, + "loss_delivery": 0.7381080447419269, + "loss_biodist": 0.45199419920508926, + "loss_toxic": 0.21455221227999474, + "loss_moe_lb": 1.0222684322176754 + }, + { + "loss": 3.5485111829396843, + "loss_size": 0.8087965942717887, + "loss_pdi": 0.6098863393873781, + "loss_ee": 0.889092169097952, + "loss_delivery": 0.8076809731286925, + "loss_biodist": 0.30694895056453914, + "loss_toxic": 0.15926390459345705, + "loss_moe_lb": 1.0093126087575346 + }, + { + "loss": 3.0572193087758244, + "loss_size": 0.48312163997340846, + "loss_pdi": 0.5788968028248968, + "loss_ee": 0.8426912733026453, + "loss_delivery": 0.7541565317157153, + "loss_biodist": 0.24662724942774386, + "loss_toxic": 0.18097399132376588, + "loss_moe_lb": 1.0077822981653988 + }, + { + "loss": 3.1274955304893286, + "loss_size": 0.7246596015788414, + "loss_pdi": 0.5637430741980269, + "loss_ee": 0.7838426683400128, + "loss_delivery": 0.7544126525722645, + "loss_biodist": 0.25151420203415126, + "loss_toxic": 0.08561256998655153, + "loss_moe_lb": 1.0103606887765832 + }, + { + "loss": 2.9605907749485327, + "loss_size": 0.7185749713633511, + "loss_pdi": 0.5364950689915065, + "loss_ee": 0.793822729104274, + "loss_delivery": 0.6642366660809195, + "loss_biodist": 0.21504729578422532, + "loss_toxic": 0.07086705067430975, + "loss_moe_lb": 1.0071579105145223 + }, + { + "loss": 2.8486427133147783, + "loss_size": 0.5915605695666494, + "loss_pdi": 0.4951009484561714, + "loss_ee": 0.7438240590933207, + "loss_delivery": 0.7274097051169421, + "loss_biodist": 0.2264139105540675, + "loss_toxic": 0.09920276786087433, + "loss_moe_lb": 1.0096762695827999 + }, + { + "loss": 2.687680244445801, + "loss_size": 0.5851070987614425, + "loss_pdi": 0.5009385371530378, + "loss_ee": 0.7555800684400507, + "loss_delivery": 0.636992874077043, + "loss_biodist": 0.158131275023963, + "loss_toxic": 0.08684718487760669, + "loss_moe_lb": 1.007036805152893 + }, + { + "loss": 2.616856671668388, + "loss_size": 0.6355135426730723, + "loss_pdi": 0.47540898621082306, + "loss_ee": 0.7055847717298044, + "loss_delivery": 0.5933513578933638, + "loss_biodist": 0.16942888447964513, + "loss_toxic": 0.06947957772157483, + "loss_moe_lb": 1.0077386240701418 + } + ], + "val": [ + { + "loss": 7.661384916305542, + "loss_size": 3.405784809589386, + "loss_pdi": 0.6896121621131897, + "loss_ee": 1.0416743278503418, + "loss_delivery": 1.717668344080448, + "loss_biodist": 0.6260749816894531, + "loss_toxic": 0.2114003360271454, + "loss_moe_lb": 1.0182087659835815, + "acc_pdi": 0.6060606060606061, + "acc_ee": 0.5454545454545454, + "acc_toxic": 1.0 + }, + { + "loss": 7.001102876663208, + "loss_size": 3.346554321050644, + "loss_pdi": 0.6230870246887207, + "loss_ee": 0.9340021252632141, + "loss_delivery": 1.6679020524024963, + "loss_biodist": 0.4111012101173401, + "loss_toxic": 0.07045397134497762, + "loss_moe_lb": 1.034538781642914, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.95 + }, + { + "loss": 6.759282732009888, + "loss_size": 3.3599622368812563, + "loss_pdi": 0.5901522099971771, + "loss_ee": 0.9094328999519348, + "loss_delivery": 1.6522429764270783, + "loss_biodist": 0.29448634684085845, + "loss_toxic": 0.02089481777511537, + "loss_moe_lb": 1.0319836854934692, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.6060606060606061, + "acc_toxic": 1.0 + }, + { + "loss": 6.69711389541626, + "loss_size": 3.333434188365936, + "loss_pdi": 0.6802905142307282, + "loss_ee": 0.9726977825164795, + "loss_delivery": 1.6111902564764022, + "loss_biodist": 0.1274224802851677, + "loss_toxic": 0.05617362331831828, + "loss_moe_lb": 1.0247936487197875, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.6363636363636364, + "acc_toxic": 0.95 + }, + { + "loss": 6.891025829315185, + "loss_size": 3.3954268366098406, + "loss_pdi": 0.6523886084556579, + "loss_ee": 1.0137632250785829, + "loss_delivery": 1.7531357824802398, + "loss_biodist": 0.14888265430927278, + "loss_toxic": 0.027145402447786182, + "loss_moe_lb": 1.0329432725906371, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.6060606060606061, + "acc_toxic": 1.0 + }, + { + "loss": 6.463539695739746, + "loss_size": 3.117310556769371, + "loss_pdi": 0.5904108643531799, + "loss_ee": 0.9055852174758912, + "loss_delivery": 1.7860299408435822, + "loss_biodist": 0.12173628807067871, + "loss_toxic": 0.036232783531886525, + "loss_moe_lb": 1.0186227560043335, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.6666666666666666, + "acc_toxic": 0.95 + }, + { + "loss": 6.321356010437012, + "loss_size": 3.142138013243675, + "loss_pdi": 0.6349482238292694, + "loss_ee": 0.9170174360275268, + "loss_delivery": 1.562937891483307, + "loss_biodist": 0.11433611214160919, + "loss_toxic": 0.04529684688313864, + "loss_moe_lb": 1.0187309384346008, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.6363636363636364, + "acc_toxic": 0.95 + }, + { + "loss": 6.308101606369019, + "loss_size": 3.082833522558212, + "loss_pdi": 0.6179247200489044, + "loss_ee": 0.9234862565994263, + "loss_delivery": 1.619089710712433, + "loss_biodist": 0.11304089352488518, + "loss_toxic": 0.048584981232124846, + "loss_moe_lb": 1.0251393556594848, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.6363636363636364, + "acc_toxic": 0.95 + }, + { + "loss": 5.993373537063599, + "loss_size": 2.7720276534557344, + "loss_pdi": 0.635417252779007, + "loss_ee": 0.9668270409107208, + "loss_delivery": 1.5715352505445481, + "loss_biodist": 0.11737952530384063, + "loss_toxic": 0.029923580681497697, + "loss_moe_lb": 1.0337027072906495, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.6363636363636364, + "acc_toxic": 0.95 + }, + { + "loss": 6.035852098464966, + "loss_size": 2.8085797876119614, + "loss_pdi": 0.6167563140392304, + "loss_ee": 0.9475209593772889, + "loss_delivery": 1.6233876049518585, + "loss_biodist": 0.11243854537606239, + "loss_toxic": 0.026505211762560064, + "loss_moe_lb": 1.025527799129486, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.95 + } + ] +} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_2/splits.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_2/splits.json new file mode 100644 index 0000000..14df306 --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_2/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 36, 37, 38, 39, 40, 41, 43, 44, 45, 47, 48, 49, 50, 51, 55, 56, 57, 58, 59, 60, 61, 63, 64, 65, 68, 69, 70, 72, 74, 76, 78, 79, 80, 82, 83, 84, 85, 86, 87, 88, 91, 93, 94, 95, 96, 97, 100, 101, 102, 103, 104, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 142, 143, 145, 147, 148, 149, 150, 151, 152, 153, 156, 157, 158, 159, 160, 161, 162, 163, 165, 166, 167, 168, 170, 171, 172, 173, 174, 175, 177, 179, 180, 181, 183, 184, 185, 186, 187, 188, 191, 192, 193, 194, 196, 197, 199, 200, 202, 203, 204, 205, 207, 208, 209, 210, 211, 212, 213, 214, 215, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 232, 233, 235, 236, 237, 238, 239, 240, 241, 242, 243, 245, 246, 249, 250, 251, 252, 253, 254, 255, 256, 257, 259, 260, 261, 263, 264, 265, 266, 267, 268, 269, 271, 272, 273, 274, 275, 276, 280, 281, 282, 283, 284, 285, 287, 289, 290, 293, 294, 295, 296, 297, 298, 299, 300, 302, 303, 305, 307, 308, 309, 310, 311, 312, 313, 314, 315, 317, 318, 320, 321, 322, 323, 324, 326, 327, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 358, 360, 361, 363, 364, 365, 366, 368, 369, 370, 371, 372, 373, 376, 377, 378, 379, 380, 381, 382, 383, 385, 386, 387, 388, 389, 390, 392, 394, 395, 398, 399, 400, 401, 402, 404, 405, 407, 408, 409, 410, 412, 413, 414, 415, 416, 417, 418], "outer_test_idx": [10, 12, 15, 31, 35, 42, 46, 52, 53, 54, 62, 66, 67, 71, 73, 75, 77, 81, 89, 90, 92, 98, 99, 105, 123, 124, 141, 144, 146, 154, 155, 164, 169, 176, 178, 182, 189, 190, 195, 198, 201, 206, 216, 217, 218, 219, 231, 234, 244, 247, 248, 258, 262, 270, 277, 278, 279, 286, 288, 291, 292, 301, 304, 306, 316, 319, 325, 328, 343, 357, 359, 362, 367, 374, 375, 384, 391, 393, 396, 397, 403, 406, 411, 419]} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_2/test_metrics.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_2/test_metrics.json new file mode 100644 index 0000000..a763c76 --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_2/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.3574223221771879, + "rmse": 0.5978480761675059, + "mae": 0.44976320376043166, + "r2": 0.2940972997529061 + }, + "delivery": { + "n_samples": 60, + "mse": 0.6797720885553556, + "rmse": 0.8244829219306824, + "mae": 0.5965622533811257, + "r2": 0.0951055293826022 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7023809523809523, + "precision": 0.6503347534996957, + "recall": 0.6810850439882699, + "f1": 0.6554552912223134 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6358640855011091, + "recall": 0.7037815126050421, + "f1": 0.646421950315165 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9508196721311475, + "precision": 0.75, + "recall": 0.9741379310344828, + "f1": 0.8200589970501475 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.14821293243934233, + "js_divergence": 0.03462949947575652 + } +} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_3/best_params.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_3/best_params.json new file mode 100644 index 0000000..48a4372 --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_3/best_params.json @@ -0,0 +1,16 @@ +{ + "dropout": 0.2277771320023231, + "lr": 0.0009696339767409517, + "weight_decay": 0.0003933823864828228, + "backbone_lr_ratio": 0.4073135560471746, + "moe_n_experts": 4, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_3/epoch_mean.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_3/epoch_mean.json new file mode 100644 index 0000000..84179ef --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_3/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 12} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_3/history.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_3/history.json new file mode 100644 index 0000000..725f48f --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_3/history.json @@ -0,0 +1,282 @@ +{ + "train": [ + { + "loss": 4.635187567891301, + "loss_size": 0.5012874734160062, + "loss_pdi": 0.6885242816564199, + "loss_ee": 1.027883595711476, + "loss_delivery": 0.9665894249605166, + "loss_biodist": 0.9150939446848791, + "loss_toxic": 0.5288639735128429, + "loss_moe_lb": 2.080437035173983 + }, + { + "loss": 3.515928548735541, + "loss_size": 0.47634934664175316, + "loss_pdi": 0.626040541642421, + "loss_ee": 0.9337553228880908, + "loss_delivery": 0.8161725210579666, + "loss_biodist": 0.409263873422468, + "loss_toxic": 0.256397028208232, + "loss_moe_lb": 2.0407471656799316 + }, + { + "loss": 3.014289211582493, + "loss_size": 0.4496709112380002, + "loss_pdi": 0.5760402929138493, + "loss_ee": 0.8493540335345913, + "loss_delivery": 0.8272613544721861, + "loss_biodist": 0.27485793093974525, + "loss_toxic": 0.05672638175530812, + "loss_moe_lb": 2.163790261423266 + }, + { + "loss": 3.191360882810644, + "loss_size": 0.5026504228243956, + "loss_pdi": 0.5937461103941943, + "loss_ee": 0.857080619882893, + "loss_delivery": 0.827085596886841, + "loss_biodist": 0.22913366616577716, + "loss_toxic": 0.21178777444859137, + "loss_moe_lb": 2.303922440554645 + }, + { + "loss": 2.9948180430644267, + "loss_size": 0.4510602429509163, + "loss_pdi": 0.5194555303534946, + "loss_ee": 0.7748511320835835, + "loss_delivery": 0.920670948318533, + "loss_biodist": 0.28029138292815237, + "loss_toxic": 0.09228010700364295, + "loss_moe_lb": 2.371367087235322 + }, + { + "loss": 2.7870203353263237, + "loss_size": 0.44745276646839605, + "loss_pdi": 0.4979797533235034, + "loss_ee": 0.77558530101905, + "loss_delivery": 0.7112665847950691, + "loss_biodist": 0.1983197205574126, + "loss_toxic": 0.19586172337703583, + "loss_moe_lb": 2.266146360216914 + }, + { + "loss": 2.5810787323358895, + "loss_size": 0.39961432967637034, + "loss_pdi": 0.5001044523071598, + "loss_ee": 0.7730238453761952, + "loss_delivery": 0.6902468486412151, + "loss_biodist": 0.20221736241836805, + "loss_toxic": 0.0798912753502058, + "loss_moe_lb": 2.2518503408174255 + }, + { + "loss": 2.328119616250734, + "loss_size": 0.3802749446114978, + "loss_pdi": 0.4632198468253419, + "loss_ee": 0.7083250831913304, + "loss_delivery": 0.6365419846732874, + "loss_biodist": 0.18871023002508525, + "loss_toxic": 0.02188232724998867, + "loss_moe_lb": 2.343865864985698 + }, + { + "loss": 2.2661313398464307, + "loss_size": 0.3468433250043843, + "loss_pdi": 0.48131870377708125, + "loss_ee": 0.6717699242604745, + "loss_delivery": 0.630864675181943, + "loss_biodist": 0.16674514804538842, + "loss_toxic": 0.04373257501031952, + "loss_moe_lb": 2.2291988939852327 + }, + { + "loss": 2.123405308336825, + "loss_size": 0.3591880999706887, + "loss_pdi": 0.4031320292804692, + "loss_ee": 0.6463552214003898, + "loss_delivery": 0.5338046378581911, + "loss_biodist": 0.1322294191734211, + "loss_toxic": 0.11041431584787977, + "loss_moe_lb": 2.259832085789861 + }, + { + "loss": 1.9465130116488483, + "loss_size": 0.30107833304115245, + "loss_pdi": 0.4084460143301938, + "loss_ee": 0.6298204736935126, + "loss_delivery": 0.544113476634831, + "loss_biodist": 0.1274531402096555, + "loss_toxic": 0.01666082850106004, + "loss_moe_lb": 2.2224921181395247 + }, + { + "loss": 1.9182801278861794, + "loss_size": 0.20874326635856885, + "loss_pdi": 0.4383741407781034, + "loss_ee": 0.6381029824952822, + "loss_delivery": 0.5332832222905111, + "loss_biodist": 0.15056242119218852, + "loss_toxic": 0.03362102670928072, + "loss_moe_lb": 2.304750532717318 + } + ], + "val": [ + { + "loss": 5.488530898094178, + "loss_size": 2.7204876095056534, + "loss_pdi": 0.695596182346344, + "loss_ee": 0.7013792157173157, + "loss_delivery": 0.9113651782274246, + "loss_biodist": 0.29361158311367036, + "loss_toxic": 0.11685825288295745, + "loss_moe_lb": 2.246588706970215, + "acc_pdi": 0.48484848484848486, + "acc_ee": 0.7272727272727273, + "acc_toxic": 1.0 + }, + { + "loss": 5.301105284690857, + "loss_size": 2.6756161481142042, + "loss_pdi": 0.6311582118272782, + "loss_ee": 0.7636628031730652, + "loss_delivery": 0.7813698470592498, + "loss_biodist": 0.19865677058696746, + "loss_toxic": 0.1989521250128746, + "loss_moe_lb": 2.393324637413025, + "acc_pdi": 0.7878787878787878, + "acc_ee": 0.696969696969697, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 4.73776638507843, + "loss_size": 2.522054509818554, + "loss_pdi": 0.5711286991834641, + "loss_ee": 0.6519612848758698, + "loss_delivery": 0.8117783546447754, + "loss_biodist": 0.10366116762161255, + "loss_toxic": 0.0277511257911101, + "loss_moe_lb": 2.4904290676116942, + "acc_pdi": 0.7878787878787878, + "acc_ee": 0.7272727272727273, + "acc_toxic": 0.9565217391304348 + }, + { + "loss": 5.396608531475067, + "loss_size": 2.598762262240052, + "loss_pdi": 0.5691216468811036, + "loss_ee": 0.8224055141210556, + "loss_delivery": 1.1457733705639839, + "loss_biodist": 0.15846161544322968, + "loss_toxic": 0.07480475245974957, + "loss_moe_lb": 2.5834170818328857, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.9565217391304348 + }, + { + "loss": 4.852854633331299, + "loss_size": 2.3418304204940794, + "loss_pdi": 0.59192253947258, + "loss_ee": 0.8005214691162109, + "loss_delivery": 0.8337671726942062, + "loss_biodist": 0.1642390102148056, + "loss_toxic": 0.09418852836824954, + "loss_moe_lb": 2.766391897201538, + "acc_pdi": 0.6060606060606061, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.9565217391304348 + }, + { + "loss": 4.77590611577034, + "loss_size": 2.560737869143486, + "loss_pdi": 0.504337415099144, + "loss_ee": 0.7301448971033097, + "loss_delivery": 0.8419133841991424, + "loss_biodist": 0.10303926989436149, + "loss_toxic": 0.005996480048634112, + "loss_moe_lb": 2.6296823024749756, + "acc_pdi": 0.8181818181818182, + "acc_ee": 0.5757575757575758, + "acc_toxic": 1.0 + }, + { + "loss": 4.333458149433136, + "loss_size": 1.9921601206064223, + "loss_pdi": 0.5920661628246308, + "loss_ee": 0.8726371824741364, + "loss_delivery": 0.7100727267563343, + "loss_biodist": 0.16265348643064498, + "loss_toxic": 0.00967565217288211, + "loss_moe_lb": 2.64205117225647, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.6666666666666666, + "acc_toxic": 1.0 + }, + { + "loss": 4.366401934623719, + "loss_size": 2.1694664672017097, + "loss_pdi": 0.545233029127121, + "loss_ee": 0.844232153892517, + "loss_delivery": 0.6349818819202483, + "loss_biodist": 0.11941656917333603, + "loss_toxic": 0.042344605177640916, + "loss_moe_lb": 2.6210580348968504, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.9565217391304348 + }, + { + "loss": 4.2258431792259215, + "loss_size": 2.1267185762524603, + "loss_pdi": 0.4950913190841675, + "loss_ee": 0.7830321550369262, + "loss_delivery": 0.6957229539752007, + "loss_biodist": 0.09492937624454498, + "loss_toxic": 0.01870271550724283, + "loss_moe_lb": 2.6808515071868895, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.6363636363636364, + "acc_toxic": 1.0 + }, + { + "loss": 4.266985666751862, + "loss_size": 2.1308643519878387, + "loss_pdi": 0.511027204990387, + "loss_ee": 0.8321340948343277, + "loss_delivery": 0.6703791484236717, + "loss_biodist": 0.0952694907784462, + "loss_toxic": 0.019628445152193308, + "loss_moe_lb": 2.67083740234375, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.6060606060606061, + "acc_toxic": 1.0 + }, + { + "loss": 4.101023435592651, + "loss_size": 2.0085878551006315, + "loss_pdi": 0.5103982090950012, + "loss_ee": 0.82158322930336, + "loss_delivery": 0.6185474842786789, + "loss_biodist": 0.1018165335059166, + "loss_toxic": 0.0338226294144988, + "loss_moe_lb": 2.640282964706421, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.9565217391304348 + }, + { + "loss": 4.09249838590622, + "loss_size": 1.9993868201971055, + "loss_pdi": 0.5254472345113754, + "loss_ee": 0.8253367006778717, + "loss_delivery": 0.6229568749666214, + "loss_biodist": 0.09972210973501205, + "loss_toxic": 0.017385368049144746, + "loss_moe_lb": 2.642714786529541, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.5757575757575758, + "acc_toxic": 1.0 + } + ] +} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_3/splits.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_3/splits.json new file mode 100644 index 0000000..7a2b13e --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_3/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [0, 1, 2, 4, 5, 7, 8, 9, 10, 11, 12, 14, 15, 16, 17, 20, 21, 22, 23, 24, 25, 27, 28, 29, 30, 31, 32, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 45, 46, 47, 51, 52, 53, 54, 55, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 73, 74, 75, 76, 77, 78, 79, 81, 82, 83, 84, 85, 86, 87, 89, 90, 91, 92, 93, 95, 96, 97, 98, 99, 100, 102, 104, 105, 106, 109, 110, 111, 112, 113, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 127, 129, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 146, 147, 148, 150, 151, 152, 153, 154, 155, 156, 158, 159, 160, 161, 162, 163, 164, 165, 166, 168, 169, 170, 172, 173, 174, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 189, 190, 191, 192, 193, 194, 195, 198, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 212, 213, 214, 215, 216, 217, 218, 219, 221, 223, 224, 226, 227, 228, 230, 231, 233, 234, 235, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 251, 252, 253, 254, 255, 256, 257, 258, 259, 262, 263, 264, 265, 266, 267, 268, 269, 270, 272, 273, 274, 275, 276, 277, 278, 279, 280, 282, 284, 286, 288, 289, 290, 291, 292, 293, 294, 295, 297, 298, 299, 301, 302, 304, 305, 306, 307, 308, 309, 310, 312, 314, 315, 316, 317, 319, 320, 321, 325, 326, 327, 328, 331, 332, 334, 336, 340, 341, 343, 344, 346, 347, 348, 350, 352, 353, 357, 359, 360, 361, 362, 363, 364, 366, 367, 368, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 382, 383, 384, 385, 386, 387, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 400, 401, 402, 403, 404, 406, 407, 410, 411, 412, 414, 415, 416, 418, 419], "outer_test_idx": [3, 6, 13, 18, 19, 26, 33, 44, 48, 49, 50, 56, 57, 72, 80, 88, 94, 101, 103, 107, 108, 114, 126, 128, 130, 145, 149, 157, 167, 171, 175, 188, 196, 197, 199, 211, 220, 222, 225, 229, 232, 236, 250, 260, 261, 271, 281, 283, 285, 287, 296, 300, 303, 311, 313, 318, 322, 323, 324, 329, 330, 333, 335, 337, 338, 339, 342, 345, 349, 351, 354, 355, 356, 358, 365, 369, 381, 388, 399, 405, 408, 409, 413, 417]} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_3/test_metrics.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_3/test_metrics.json new file mode 100644 index 0000000..d0926b2 --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_3/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 83, + "mse": 1.6763180404940512, + "rmse": 1.294727013889048, + "mae": 0.5597611944561443, + "r2": 0.051861739559906805 + }, + "delivery": { + "n_samples": 59, + "mse": 0.6338853934116598, + "rmse": 0.7961691989845248, + "mae": 0.5351308236151175, + "r2": 0.3837880851060248 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6072124756335282, + "recall": 0.6209677419354839, + "f1": 0.6110444177671068 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5833333333333334, + "precision": 0.522765246449457, + "recall": 0.5463298139768727, + "f1": 0.5215104103741292 + }, + "toxic": { + "n_samples": 60, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.15291185556789416, + "js_divergence": 0.03702706983171318 + } +} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_4/best_params.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_4/best_params.json new file mode 100644 index 0000000..c6208a9 --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_4/best_params.json @@ -0,0 +1,16 @@ +{ + "dropout": 0.3336459802315832, + "lr": 0.0005171929496634874, + "weight_decay": 0.00014235198757319405, + "backbone_lr_ratio": 0.23125566110577128, + "moe_n_experts": 4, + "moe_top_k": 1, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_4/epoch_mean.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_4/epoch_mean.json new file mode 100644 index 0000000..54e2f4a --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_4/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 15} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_4/history.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_4/history.json new file mode 100644 index 0000000..729d49c --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_4/history.json @@ -0,0 +1,351 @@ +{ + "train": [ + { + "loss": 5.63419669383281, + "loss_size": 1.1602854293745917, + "loss_pdi": 0.6989228258261809, + "loss_ee": 1.066945399786975, + "loss_delivery": 1.0121343750003222, + "loss_biodist": 1.0827299034273303, + "loss_toxic": 0.6116908885337211, + "loss_moe_lb": 1.0093251627844733 + }, + { + "loss": 4.748308052887788, + "loss_size": 1.1241863587418117, + "loss_pdi": 0.6578884398615038, + "loss_ee": 0.9946177472939363, + "loss_delivery": 0.9186682015456058, + "loss_biodist": 0.6621862412304491, + "loss_toxic": 0.39953387830708476, + "loss_moe_lb": 1.0048162631086401 + }, + { + "loss": 4.14961728534183, + "loss_size": 1.067172997102544, + "loss_pdi": 0.6364535109416859, + "loss_ee": 0.8928145154102428, + "loss_delivery": 0.8879460586688003, + "loss_biodist": 0.420376545271358, + "loss_toxic": 0.25793624354677425, + "loss_moe_lb": 1.0095633316684414 + }, + { + "loss": 3.9389670571765385, + "loss_size": 1.0701550154669865, + "loss_pdi": 0.6052323950303568, + "loss_ee": 0.8723501337541116, + "loss_delivery": 0.8260172088001225, + "loss_biodist": 0.3739219839508469, + "loss_toxic": 0.21106538686246887, + "loss_moe_lb": 1.0101642334783398 + }, + { + "loss": 3.412070931615056, + "loss_size": 0.9763722574791392, + "loss_pdi": 0.5563612242002745, + "loss_ee": 0.7801438795553671, + "loss_delivery": 0.7706274415391523, + "loss_biodist": 0.2709137654385051, + "loss_toxic": 0.0769927189307841, + "loss_moe_lb": 1.0056231118537284 + }, + { + "loss": 3.2564174516780957, + "loss_size": 0.9089559580425959, + "loss_pdi": 0.5454743480360186, + "loss_ee": 0.7711224104907062, + "loss_delivery": 0.7078545463044901, + "loss_biodist": 0.25039478049084946, + "loss_toxic": 0.09474448292245585, + "loss_moe_lb": 1.0091406364698667 + }, + { + "loss": 3.207786650271029, + "loss_size": 0.964230006611025, + "loss_pdi": 0.5113354430005357, + "loss_ee": 0.734847777598613, + "loss_delivery": 0.6857613865789529, + "loss_biodist": 0.2302984739276203, + "loss_toxic": 0.10459642371515164, + "loss_moe_lb": 1.0037095063441508 + }, + { + "loss": 2.9567287000449927, + "loss_size": 0.8148566746228451, + "loss_pdi": 0.48902239630351196, + "loss_ee": 0.7666944651990324, + "loss_delivery": 0.6774945171499575, + "loss_biodist": 0.18696938817565506, + "loss_toxic": 0.05218754382096105, + "loss_moe_lb": 1.0103857678336066 + }, + { + "loss": 2.849684164330766, + "loss_size": 0.7122420634369593, + "loss_pdi": 0.47219982662716425, + "loss_ee": 0.6587362152499121, + "loss_delivery": 0.6606852850197135, + "loss_biodist": 0.19918360681952657, + "loss_toxic": 0.16703398835159386, + "loss_moe_lb": 1.0030312554256335 + }, + { + "loss": 2.7750655283799044, + "loss_size": 0.7347101832966547, + "loss_pdi": 0.4771474982435639, + "loss_ee": 0.629725489261988, + "loss_delivery": 0.6903010483931851, + "loss_biodist": 0.19950283963132548, + "loss_toxic": 0.07284470700117684, + "loss_moe_lb": 1.0072322568377934 + }, + { + "loss": 2.5980582575540283, + "loss_size": 0.7259641351127947, + "loss_pdi": 0.4865803025864266, + "loss_ee": 0.609408181664106, + "loss_delivery": 0.5581716090642117, + "loss_biodist": 0.1830760760081781, + "loss_toxic": 0.06320205566825697, + "loss_moe_lb": 1.006496912724263 + }, + { + "loss": 2.5412514016434953, + "loss_size": 0.6597443964030292, + "loss_pdi": 0.42990235058037013, + "loss_ee": 0.6397995634658916, + "loss_delivery": 0.6235348982988177, + "loss_biodist": 0.1862178577764614, + "loss_toxic": 0.03553678431057346, + "loss_moe_lb": 1.012873523944133 + }, + { + "loss": 2.438081654342445, + "loss_size": 0.5424814461856275, + "loss_pdi": 0.4223199913630614, + "loss_ee": 0.6402273037143655, + "loss_delivery": 0.6308092774974333, + "loss_biodist": 0.17005812195507256, + "loss_toxic": 0.06458786742547779, + "loss_moe_lb": 1.0040894634014852 + }, + { + "loss": 2.4209886306040995, + "loss_size": 0.578826969942531, + "loss_pdi": 0.4276882824865547, + "loss_ee": 0.5754776045277312, + "loss_delivery": 0.614098277647753, + "loss_biodist": 0.19075405396319725, + "loss_toxic": 0.06323446214220305, + "loss_moe_lb": 1.0079342658455308 + }, + { + "loss": 2.6601416774698206, + "loss_size": 0.7193897281949585, + "loss_pdi": 0.4460759887824187, + "loss_ee": 0.6096655106222307, + "loss_delivery": 0.626142755553529, + "loss_biodist": 0.18131303545590993, + "loss_toxic": 0.10707091731239564, + "loss_moe_lb": 0.9982752558347341 + } + ], + "val": [ + { + "loss": 4.545587205886841, + "loss_size": 0.560679578781128, + "loss_pdi": 0.6517605543136596, + "loss_ee": 0.9948954105377197, + "loss_delivery": 0.8593550998717546, + "loss_biodist": 0.9254570007324219, + "loss_toxic": 0.56278977394104, + "loss_moe_lb": 1.0498411655426025, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 3.6502355575561523, + "loss_size": 0.5132984548807145, + "loss_pdi": 0.5906826376914978, + "loss_ee": 0.841532826423645, + "loss_delivery": 0.7773929027840495, + "loss_biodist": 0.5494850218296051, + "loss_toxic": 0.38314359188079833, + "loss_moe_lb": 1.045540189743042, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 3.278236675262451, + "loss_size": 0.47074146568775177, + "loss_pdi": 0.5141007274389267, + "loss_ee": 0.8264788389205933, + "loss_delivery": 0.7893360061563726, + "loss_biodist": 0.26280554831027986, + "loss_toxic": 0.4181690584868193, + "loss_moe_lb": 1.057166290283203, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 3.4404584407806396, + "loss_size": 0.37577483952045443, + "loss_pdi": 0.5006685696542263, + "loss_ee": 0.8671600878238678, + "loss_delivery": 1.045976811647415, + "loss_biodist": 0.12762191519141197, + "loss_toxic": 0.5316818872466683, + "loss_moe_lb": 1.067996096611023, + "acc_pdi": 0.5757575757575758, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 3.1310470581054686, + "loss_size": 0.2975272566080093, + "loss_pdi": 0.455416277050972, + "loss_ee": 0.8459741249680519, + "loss_delivery": 0.89022017121315, + "loss_biodist": 0.10563880652189254, + "loss_toxic": 0.5401085783727467, + "loss_moe_lb": 1.0564915895462037, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 3.18749258518219, + "loss_size": 0.37602749466896057, + "loss_pdi": 0.4259258806705475, + "loss_ee": 0.8476248443126678, + "loss_delivery": 0.8920384079217911, + "loss_biodist": 0.13155047744512557, + "loss_toxic": 0.52066456861794, + "loss_moe_lb": 1.0411512851715088, + "acc_pdi": 0.8181818181818182, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 2.8350556612014772, + "loss_size": 0.2839426174759865, + "loss_pdi": 0.4367354616522789, + "loss_ee": 0.8004509478807449, + "loss_delivery": 0.8504814647138119, + "loss_biodist": 0.13076556846499443, + "loss_toxic": 0.346000572713092, + "loss_moe_lb": 1.033005154132843, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 2.7681538701057433, + "loss_size": 0.2955185487866402, + "loss_pdi": 0.42818596214056015, + "loss_ee": 0.81761210039258, + "loss_delivery": 0.9430503427982331, + "loss_biodist": 0.10055616497993469, + "loss_toxic": 0.2107411513570696, + "loss_moe_lb": 1.0307934284210205, + "acc_pdi": 0.7878787878787878, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.876146066188812, + "loss_size": 0.3166431590914726, + "loss_pdi": 0.4234211968258023, + "loss_ee": 0.8333672847598791, + "loss_delivery": 1.0074312508106231, + "loss_biodist": 0.11880894862115383, + "loss_toxic": 0.21037241583690047, + "loss_moe_lb": 1.0347557544708252, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.851035180687904, + "loss_size": 0.3158852785825729, + "loss_pdi": 0.4339466694742441, + "loss_ee": 0.8876120168715715, + "loss_delivery": 1.0336736522614955, + "loss_biodist": 0.09949213489890099, + "loss_toxic": 0.1277771416120231, + "loss_moe_lb": 1.0263399720191955, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.7765939831733704, + "loss_size": 0.24978366121649742, + "loss_pdi": 0.42311569638550284, + "loss_ee": 0.8873283516615629, + "loss_delivery": 1.0131873220205307, + "loss_biodist": 0.1037889838218689, + "loss_toxic": 0.14672286827117204, + "loss_moe_lb": 1.0274889707565307, + "acc_pdi": 0.7878787878787878, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.8117101073265074, + "loss_size": 0.3265482311602682, + "loss_pdi": 0.43741046749055384, + "loss_ee": 0.8583829991519452, + "loss_delivery": 0.9985359662212432, + "loss_biodist": 0.10368301793932914, + "loss_toxic": 0.13591801449656488, + "loss_moe_lb": 1.0229694366455078, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.8324283987283705, + "loss_size": 0.3066274739103392, + "loss_pdi": 0.4413070822134614, + "loss_ee": 0.8623994909226894, + "loss_delivery": 0.9875150389969349, + "loss_biodist": 0.10742180570960044, + "loss_toxic": 0.1741462097503245, + "loss_moe_lb": 1.01354261636734, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.7185962200164795, + "loss_size": 0.2421225077821873, + "loss_pdi": 0.43503497298806904, + "loss_ee": 0.8452146667987108, + "loss_delivery": 0.9819117061793804, + "loss_biodist": 0.10673428922891617, + "loss_toxic": 0.15479623645078391, + "loss_moe_lb": 1.0309493780136108, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.673775205016136, + "loss_size": 0.2686118818819523, + "loss_pdi": 0.4386091554537416, + "loss_ee": 0.8537979789078236, + "loss_delivery": 0.884277717396617, + "loss_biodist": 0.10967024266719819, + "loss_toxic": 0.16369831073097885, + "loss_moe_lb": 1.0288986563682556, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.9444444444444444 + } + ] +} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_4/splits.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_4/splits.json new file mode 100644 index 0000000..8dd0299 --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_4/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23, 25, 26, 28, 29, 31, 33, 34, 35, 37, 41, 42, 43, 44, 45, 46, 48, 49, 50, 51, 52, 53, 54, 56, 57, 59, 60, 62, 63, 64, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 77, 78, 79, 80, 81, 82, 83, 84, 85, 87, 88, 89, 90, 91, 92, 93, 94, 96, 97, 98, 99, 101, 102, 103, 104, 105, 107, 108, 110, 111, 112, 113, 114, 116, 117, 119, 120, 121, 122, 123, 124, 126, 127, 128, 130, 132, 133, 134, 135, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 149, 152, 154, 155, 157, 160, 161, 162, 163, 164, 165, 166, 167, 169, 170, 171, 173, 174, 175, 176, 177, 178, 180, 181, 182, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 203, 205, 206, 207, 208, 209, 211, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 229, 230, 231, 232, 234, 235, 236, 238, 239, 240, 241, 243, 244, 245, 246, 247, 248, 249, 250, 251, 253, 254, 257, 258, 260, 261, 262, 263, 264, 265, 266, 267, 270, 271, 272, 273, 277, 278, 279, 280, 281, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 298, 299, 300, 301, 302, 303, 304, 305, 306, 309, 311, 312, 313, 314, 315, 316, 318, 319, 320, 322, 323, 324, 325, 326, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 341, 342, 343, 345, 346, 347, 349, 350, 351, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 365, 367, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 381, 383, 384, 385, 386, 388, 390, 391, 392, 393, 394, 395, 396, 397, 399, 400, 401, 402, 403, 405, 406, 407, 408, 409, 410, 411, 413, 415, 416, 417, 418, 419], "outer_test_idx": [0, 5, 11, 21, 24, 27, 30, 32, 36, 38, 39, 40, 47, 55, 58, 61, 65, 76, 86, 95, 100, 106, 109, 115, 118, 125, 129, 131, 136, 148, 150, 151, 153, 156, 158, 159, 168, 172, 179, 183, 184, 185, 202, 204, 210, 212, 213, 228, 233, 237, 242, 252, 255, 256, 259, 268, 269, 274, 275, 276, 282, 297, 307, 308, 310, 317, 321, 327, 340, 344, 348, 352, 364, 366, 368, 379, 380, 382, 387, 389, 398, 404, 412, 414]} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/outer_fold_4/test_metrics.json b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_4/test_metrics.json new file mode 100644 index 0000000..f45e950 --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/outer_fold_4/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.8690650468716618, + "rmse": 0.932236583100911, + "mae": 0.5353849688045946, + "r2": -0.1629169844202072 + }, + "delivery": { + "n_samples": 58, + "mse": 1.068608919216617, + "rmse": 1.0337354203163482, + "mae": 0.6252993111466539, + "r2": -0.033327184177486036 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7738095238095238, + "precision": 0.7163742690058479, + "recall": 0.7441348973607038, + "f1": 0.7262905162064826 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6666666666666666, + "precision": 0.6228070175438597, + "recall": 0.6837606837606837, + "f1": 0.6349206349206349 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.8659090909090909 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.22372876119646326, + "js_divergence": 0.0564551621929834 + } +} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/strata_info.json b/models/abl_full/s1_both_chemeleon/seed42/strata_info.json new file mode 100644 index 0000000..ef3be58 --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/strata_info.json @@ -0,0 +1,60 @@ +{ + "original_strata_counts": { + "T0|P0|E0": "5", + "T0|P0|E1": "58", + "T0|P0|E2": "169", + "T0|P1|E0": "1", + "T0|P1|E1": "17", + "T0|P1|E2": "32", + "T0|P2|E2": "3", + "T1|P0|E2": "9", + "T1|P1|E2": "5", + "TNA|P0|E0": "28", + "TNA|P0|E1": "21", + "TNA|P0|E2": "20", + "TNA|P1|E0": "29", + "TNA|P1|E1": "7", + "TNA|P1|E2": "14", + "TNA|P2|E2": "1", + "TNA|P3|E0": "1" + }, + "rare_strata": [ + "T0|P1|E0", + "T0|P2|E2", + "TNA|P2|E2", + "TNA|P3|E0" + ], + "final_strata": [ + "RARE", + "T0|P0|E0", + "T0|P0|E1", + "T0|P0|E2", + "T0|P1|E1", + "T0|P1|E2", + "T1|P0|E2", + "T1|P1|E2", + "TNA|P0|E0", + "TNA|P0|E1", + "TNA|P0|E2", + "TNA|P1|E0", + "TNA|P1|E1", + "TNA|P1|E2" + ], + "final_strata_counts": { + "RARE": "6", + "T0|P0|E0": "5", + "T0|P0|E1": "58", + "T0|P0|E2": "169", + "T0|P1|E1": "17", + "T0|P1|E2": "32", + "T1|P0|E2": "9", + "T1|P1|E2": "5", + "TNA|P0|E0": "28", + "TNA|P0|E1": "21", + "TNA|P0|E2": "20", + "TNA|P1|E0": "29", + "TNA|P1|E1": "7", + "TNA|P1|E2": "14" + }, + "n_rare_merged": "6" +} \ No newline at end of file diff --git a/models/abl_full/s1_both_chemeleon/seed42/summary.json b/models/abl_full/s1_both_chemeleon/seed42/summary.json new file mode 100644 index 0000000..ab5b554 --- /dev/null +++ b/models/abl_full/s1_both_chemeleon/seed42/summary.json @@ -0,0 +1,372 @@ +{ + "fold_results": [ + { + "fold": 0, + "best_params": { + "dropout": 0.4108948503325973, + "lr": 0.0007340472227986773, + "weight_decay": 0.0007067135047679004, + "backbone_lr_ratio": 0.7321173243252591, + "moe_n_experts": 2, + "moe_top_k": 2, + "moe_expert_hidden_mult": 1, + "llm_lora_r": 32, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 14, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.566305704940777, + "rmse": 1.2515213561664766, + "mae": 0.6106541865583823, + "r2": 0.04138235411462465 + }, + "delivery": { + "n_samples": 58, + "mse": 0.5779477061981523, + "rmse": 0.7602287196614926, + "mae": 0.6029246019777553, + "r2": 0.2653935897935683 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.75, + "precision": 0.7018518518518518, + "recall": 0.733071988595866, + "f1": 0.7105824446267432 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.75, + "precision": 0.7208073948311639, + "recall": 0.7614285714285715, + "f1": 0.7248345524207593 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9655172413793104, + "precision": 0.75, + "recall": 0.9821428571428572, + "f1": 0.8242424242424242 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.21251985213285735, + "js_divergence": 0.04386748213277705 + } + } + }, + { + "fold": 1, + "best_params": { + "dropout": 0.24482591245144014, + "lr": 0.00042458936788614755, + "weight_decay": 0.00014902327669828248, + "backbone_lr_ratio": 0.24808069123277332, + "moe_n_experts": 2, + "moe_top_k": 1, + "moe_expert_hidden_mult": 1, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 7, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.35709959292610643, + "rmse": 0.597578106130158, + "mae": 0.46698603426505414, + "r2": -0.09045519979354744 + }, + "delivery": { + "n_samples": 61, + "mse": 1.0490149027835551, + "rmse": 1.0242142855787333, + "mae": 0.6541144933062987, + "r2": 0.20597134095659064 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.5974801061007957, + "recall": 0.6111111111111112, + "f1": 0.6011957095129241 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5833333333333334, + "precision": 0.5340802987861811, + "recall": 0.5444079729794016, + "f1": 0.5238095238095238 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9508196721311475, + "precision": 0.75, + "recall": 0.9741379310344828, + "f1": 0.8200589970501475 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.35185404638391565, + "js_divergence": 0.09285944999940988 + } + } + }, + { + "fold": 2, + "best_params": { + "dropout": 0.330147494187667, + "lr": 0.0007391626870428583, + "weight_decay": 0.08248214185502709, + "backbone_lr_ratio": 0.3321485864522685, + "moe_n_experts": 4, + "moe_top_k": 1, + "moe_expert_hidden_mult": 1, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 10, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.3574223221771879, + "rmse": 0.5978480761675059, + "mae": 0.44976320376043166, + "r2": 0.2940972997529061 + }, + "delivery": { + "n_samples": 60, + "mse": 0.6797720885553556, + "rmse": 0.8244829219306824, + "mae": 0.5965622533811257, + "r2": 0.0951055293826022 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7023809523809523, + "precision": 0.6503347534996957, + "recall": 0.6810850439882699, + "f1": 0.6554552912223134 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6358640855011091, + "recall": 0.7037815126050421, + "f1": 0.646421950315165 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9508196721311475, + "precision": 0.75, + "recall": 0.9741379310344828, + "f1": 0.8200589970501475 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.14821293243934233, + "js_divergence": 0.03462949947575652 + } + } + }, + { + "fold": 3, + "best_params": { + "dropout": 0.2277771320023231, + "lr": 0.0009696339767409517, + "weight_decay": 0.0003933823864828228, + "backbone_lr_ratio": 0.4073135560471746, + "moe_n_experts": 4, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 12, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.6763180404940512, + "rmse": 1.294727013889048, + "mae": 0.5597611944561443, + "r2": 0.051861739559906805 + }, + "delivery": { + "n_samples": 59, + "mse": 0.6338853934116598, + "rmse": 0.7961691989845248, + "mae": 0.5351308236151175, + "r2": 0.3837880851060248 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6072124756335282, + "recall": 0.6209677419354839, + "f1": 0.6110444177671068 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5833333333333334, + "precision": 0.522765246449457, + "recall": 0.5463298139768727, + "f1": 0.5215104103741292 + }, + "toxic": { + "n_samples": 60, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.15291185556789416, + "js_divergence": 0.03702706983171318 + } + } + }, + { + "fold": 4, + "best_params": { + "dropout": 0.3336459802315832, + "lr": 0.0005171929496634874, + "weight_decay": 0.00014235198757319405, + "backbone_lr_ratio": 0.23125566110577128, + "moe_n_experts": 4, + "moe_top_k": 1, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 15, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.8690650468716618, + "rmse": 0.932236583100911, + "mae": 0.5353849688045946, + "r2": -0.1629169844202072 + }, + "delivery": { + "n_samples": 58, + "mse": 1.068608919216617, + "rmse": 1.0337354203163482, + "mae": 0.6252993111466539, + "r2": -0.033327184177486036 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7738095238095238, + "precision": 0.7163742690058479, + "recall": 0.7441348973607038, + "f1": 0.7262905162064826 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6666666666666666, + "precision": 0.6228070175438597, + "recall": 0.6837606837606837, + "f1": 0.6349206349206349 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.8659090909090909 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.22372876119646326, + "js_divergence": 0.0564551621929834 + } + } + } + ], + "summary_stats": { + "size": { + "mse_mean": 0.965242141481957, + "mse_std": 0.5684080694545139, + "rmse_mean": 0.9347822270908199, + "rmse_std": 0.30236456372578385, + "mae_mean": 0.5245099175689214, + "mae_std": 0.059460613378625654, + "r2_mean": 0.026793841842736588, + "r2_std": 0.15622413268313906 + }, + "delivery": { + "mse_mean": 0.801845802033068, + "mse_std": 0.2123667454006387, + "rmse_mean": 0.8877661092943562, + "rmse_std": 0.11712018280992008, + "mae_mean": 0.6028062966853902, + "mae_std": 0.0393889285669396, + "r2_mean": 0.18338627221225998, + "r2_std": 0.14293661799071766 + }, + "pdi": { + "accuracy_mean": 0.7166666666666666, + "accuracy_std": 0.03868589716493314, + "precision_mean": 0.6546506912183438, + "precision_std": 0.0481149395489297, + "recall_mean": 0.678074156598287, + "recall_std": 0.05503279112301304, + "f1_mean": 0.660913675867114, + "f1_std": 0.050644032648896935 + }, + "ee": { + "accuracy_mean": 0.6523809523809524, + "accuracy_std": 0.06317380553057902, + "precision_mean": 0.6072648086223541, + "precision_std": 0.07272901245927187, + "recall_mean": 0.6479417109501143, + "recall_std": 0.08754948574150694, + "f1_mean": 0.6102994143680424, + "f1_std": 0.07796579808167317 + }, + "toxic": { + "accuracy_mean": 0.9666516561113718, + "accuracy_std": 0.017972227617721277, + "precision_mean": 0.8099999999999999, + "precision_std": 0.09695359714832658, + "recall_mean": 0.982512315270936, + "recall_std": 0.009448309250648584, + "f1_mean": 0.866053901850362, + "f1_std": 0.06916768085550049 + }, + "biodist": { + "kl_divergence_mean": 0.21784548954409458, + "kl_divergence_std": 0.07360209023630354, + "js_divergence_mean": 0.052967732726528015, + "js_divergence_std": 0.021338797362634734 + } + } +} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_0/best_params.json b/models/abl_full/s2_mole/seed42/outer_fold_0/best_params.json new file mode 100644 index 0000000..754f135 --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_0/best_params.json @@ -0,0 +1,12 @@ +{ + "dropout": 0.10525612628712336, + "lr": 0.0008524636948952184, + "weight_decay": 0.00045548267300368544, + "backbone_lr_ratio": 0.041128435798161375, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_0/epoch_mean.json b/models/abl_full/s2_mole/seed42/outer_fold_0/epoch_mean.json new file mode 100644 index 0000000..82d35d4 --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_0/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 14} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_0/history.json b/models/abl_full/s2_mole/seed42/outer_fold_0/history.json new file mode 100644 index 0000000..73ad5fa --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_0/history.json @@ -0,0 +1,300 @@ +{ + "train": [ + { + "loss": 5.391795833905538, + "loss_size": 0.8953811956776513, + "loss_pdi": 0.6740280952718523, + "loss_ee": 1.0509012109703488, + "loss_delivery": 1.036527059144444, + "loss_biodist": 1.165412078301112, + "loss_toxic": 0.5799583709902234 + }, + { + "loss": 4.249991721577114, + "loss_size": 0.7791667576465342, + "loss_pdi": 0.6154247654808892, + "loss_ee": 0.9630435241593255, + "loss_delivery": 0.7971602492034435, + "loss_biodist": 0.7085605892870162, + "loss_toxic": 0.4050261477629344 + }, + { + "loss": 3.7685992982652454, + "loss_size": 0.7470943985713853, + "loss_pdi": 0.5794505361053679, + "loss_ee": 0.9004692104127672, + "loss_delivery": 0.8416421388586363, + "loss_biodist": 0.4308814654747645, + "loss_toxic": 0.28902267747455174 + }, + { + "loss": 3.2369043694602118, + "loss_size": 0.6307582296431065, + "loss_pdi": 0.5406960116492378, + "loss_ee": 0.854380918873681, + "loss_delivery": 0.7798494108849101, + "loss_biodist": 0.29091140379508335, + "loss_toxic": 0.16265568137168884 + }, + { + "loss": 3.103496273358663, + "loss_size": 0.6623917122681936, + "loss_pdi": 0.5170113924476836, + "loss_ee": 0.8297546539041731, + "loss_delivery": 0.7559870895412233, + "loss_biodist": 0.25918200943205094, + "loss_toxic": 0.10462813265621662 + }, + { + "loss": 2.920563757419586, + "loss_size": 0.5730272779862086, + "loss_pdi": 0.49345390167501235, + "loss_ee": 0.7673006289535098, + "loss_delivery": 0.7838828331894345, + "loss_biodist": 0.22126946308546597, + "loss_toxic": 0.10725695277667707 + }, + { + "loss": 2.7595673534605236, + "loss_size": 0.5487298493583997, + "loss_pdi": 0.4917331470383538, + "loss_ee": 0.7782071994410621, + "loss_delivery": 0.6627001977629132, + "loss_biodist": 0.21037213007609049, + "loss_toxic": 0.09512323280796409 + }, + { + "loss": 2.7403381533092923, + "loss_size": 0.5547761718432108, + "loss_pdi": 0.48127957847383285, + "loss_ee": 0.7491429911719428, + "loss_delivery": 0.6845142642656962, + "loss_biodist": 0.1941047343942854, + "loss_toxic": 0.105078448024061 + }, + { + "loss": 2.7766258451673718, + "loss_size": 0.5235511627462175, + "loss_pdi": 0.48719654149479336, + "loss_ee": 0.7618867291344537, + "loss_delivery": 0.7171630726920234, + "loss_biodist": 0.1598263560897774, + "loss_toxic": 0.15725799329164955 + }, + { + "loss": 2.703974485397339, + "loss_size": 0.5218392122122977, + "loss_pdi": 0.47683804316653144, + "loss_ee": 0.7619199156761169, + "loss_delivery": 0.6990946307778358, + "loss_biodist": 0.18696178247531256, + "loss_toxic": 0.09284896558771531 + }, + { + "loss": 2.4888089299201965, + "loss_size": 0.32745574166377384, + "loss_pdi": 0.46869397660096485, + "loss_ee": 0.7419254001643922, + "loss_delivery": 0.7050711346997155, + "loss_biodist": 0.17792756938272053, + "loss_toxic": 0.1032631258583731 + }, + { + "loss": 2.5719781716664634, + "loss_size": 0.43959251874023014, + "loss_pdi": 0.4440951827499602, + "loss_ee": 0.770681596464581, + "loss_delivery": 0.6523953850070635, + "loss_biodist": 0.1820297050807211, + "loss_toxic": 0.11808631439796752 + }, + { + "loss": 2.6111876434750028, + "loss_size": 0.44898125280936557, + "loss_pdi": 0.481866341498163, + "loss_ee": 0.7717456287807889, + "loss_delivery": 0.7028302434417937, + "loss_biodist": 0.17501405916280216, + "loss_toxic": 0.07186089963134792 + }, + { + "loss": 2.5495795408884683, + "loss_size": 0.476736918091774, + "loss_pdi": 0.46281227138307357, + "loss_ee": 0.7154201798968844, + "loss_delivery": 0.6504828168286217, + "loss_biodist": 0.16178705336319077, + "loss_toxic": 0.11548970207675463 + } + ], + "val": [ + { + "loss": 4.1776069800059, + "loss_size": 0.3983742994817779, + "loss_pdi": 0.7102998892466227, + "loss_ee": 0.9521994392077128, + "loss_delivery": 0.6353711591412624, + "loss_biodist": 0.9753743012746176, + "loss_toxic": 0.523979256550471, + "acc_pdi": 0.5454545454545454, + "acc_ee": 0.48484848484848486, + "acc_toxic": 0.9 + }, + { + "loss": 3.7187440395355225, + "loss_size": 0.3908009057243665, + "loss_pdi": 0.8424604733784994, + "loss_ee": 0.8571085929870605, + "loss_delivery": 0.5692679435014725, + "loss_biodist": 0.5800763169924418, + "loss_toxic": 0.5010223984718323, + "acc_pdi": 0.5151515151515151, + "acc_ee": 0.45454545454545453, + "acc_toxic": 0.9 + }, + { + "loss": 3.5326627095540366, + "loss_size": 0.37921073536078137, + "loss_pdi": 0.9119113087654114, + "loss_ee": 0.7930368383725485, + "loss_delivery": 0.5642277747392654, + "loss_biodist": 0.45784496267636615, + "loss_toxic": 0.4527124712864558, + "acc_pdi": 0.3939393939393939, + "acc_ee": 0.45454545454545453, + "acc_toxic": 0.9 + }, + { + "loss": 3.184494972229004, + "loss_size": 0.37303654849529266, + "loss_pdi": 1.0106977224349976, + "loss_ee": 0.7158440041045347, + "loss_delivery": 0.5037710986410578, + "loss_biodist": 0.3575533628463745, + "loss_toxic": 0.2539549693465233, + "acc_pdi": 0.45454545454545453, + "acc_ee": 0.45454545454545453, + "acc_toxic": 1.0 + }, + { + "loss": 3.317845106124878, + "loss_size": 0.4185659984747569, + "loss_pdi": 1.086433966954549, + "loss_ee": 0.6946563099821409, + "loss_delivery": 0.4396252694229285, + "loss_biodist": 0.30428655942281085, + "loss_toxic": 0.40261560181776684, + "acc_pdi": 0.5757575757575758, + "acc_ee": 0.48484848484848486, + "acc_toxic": 0.9 + }, + { + "loss": 3.3756088415781655, + "loss_size": 0.3993782301743825, + "loss_pdi": 1.1885510087013245, + "loss_ee": 0.7107109328111013, + "loss_delivery": 0.43275995055834454, + "loss_biodist": 0.2685920049746831, + "loss_toxic": 0.4087022269765536, + "acc_pdi": 0.5151515151515151, + "acc_ee": 0.45454545454545453, + "acc_toxic": 0.9 + }, + { + "loss": 2.9868388175964355, + "loss_size": 0.3775108953317006, + "loss_pdi": 1.0946298837661743, + "loss_ee": 0.7107796669006348, + "loss_delivery": 0.39517642557621, + "loss_biodist": 0.23634537806113562, + "loss_toxic": 0.20957466525336108, + "acc_pdi": 0.5757575757575758, + "acc_ee": 0.45454545454545453, + "acc_toxic": 1.0 + }, + { + "loss": 2.794677972793579, + "loss_size": 0.3364267448584239, + "loss_pdi": 1.1274744868278503, + "loss_ee": 0.6875753284742435, + "loss_delivery": 0.3990583581229051, + "loss_biodist": 0.242701455950737, + "loss_toxic": 0.048559973016381264, + "acc_pdi": 0.6060606060606061, + "acc_ee": 0.45454545454545453, + "acc_toxic": 1.0 + }, + { + "loss": 3.0297327041625977, + "loss_size": 0.4073829750219981, + "loss_pdi": 1.2665051817893982, + "loss_ee": 0.68869006074965, + "loss_delivery": 0.4177926406264305, + "loss_biodist": 0.23916998753945032, + "loss_toxic": 0.06421494173506896, + "acc_pdi": 0.5454545454545454, + "acc_ee": 0.45454545454545453, + "acc_toxic": 1.0 + }, + { + "loss": 3.1811575889587402, + "loss_size": 0.4268197814623515, + "loss_pdi": 1.3070880969365437, + "loss_ee": 0.699071587373813, + "loss_delivery": 0.4416930327812831, + "loss_biodist": 0.2384570681800445, + "loss_toxic": 0.12300986734529336, + "acc_pdi": 0.5757575757575758, + "acc_ee": 0.45454545454545453, + "acc_toxic": 1.0 + }, + { + "loss": 3.2096020380655923, + "loss_size": 0.42810918887456256, + "loss_pdi": 1.335972011089325, + "loss_ee": 0.6975946842382351, + "loss_delivery": 0.4291747423509757, + "loss_biodist": 0.2540668062865734, + "loss_toxic": 0.12125987869997819, + "acc_pdi": 0.5757575757575758, + "acc_ee": 0.45454545454545453, + "acc_toxic": 1.0 + }, + { + "loss": 3.1686011950174966, + "loss_size": 0.39861536026000977, + "loss_pdi": 1.357095996538798, + "loss_ee": 0.6952343365798394, + "loss_delivery": 0.4316791867216428, + "loss_biodist": 0.24436364447077116, + "loss_toxic": 0.10049400416513284, + "acc_pdi": 0.6060606060606061, + "acc_ee": 0.45454545454545453, + "acc_toxic": 1.0 + }, + { + "loss": 3.1865879694620767, + "loss_size": 0.4051973025004069, + "loss_pdi": 1.355741302172343, + "loss_ee": 0.6929110158234835, + "loss_delivery": 0.43002021064360935, + "loss_biodist": 0.2367245008548101, + "loss_toxic": 0.12338362944622834, + "acc_pdi": 0.5757575757575758, + "acc_ee": 0.45454545454545453, + "acc_toxic": 1.0 + }, + { + "loss": 3.145195960998535, + "loss_size": 0.4192396899064382, + "loss_pdi": 1.3251916964848836, + "loss_ee": 0.6955831789722046, + "loss_delivery": 0.41368140528599423, + "loss_biodist": 0.18375763048728308, + "loss_toxic": 0.16095475107431412, + "acc_pdi": 0.6060606060606061, + "acc_ee": 0.45454545454545453, + "acc_toxic": 1.0 + } + ] +} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_0/splits.json b/models/abl_full/s2_mole/seed42/outer_fold_0/splits.json new file mode 100644 index 0000000..cccb9d8 --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_0/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 52, 53, 54, 55, 56, 57, 58, 61, 62, 63, 64, 65, 66, 67, 68, 71, 72, 73, 75, 76, 77, 80, 81, 85, 86, 88, 89, 90, 92, 94, 95, 96, 98, 99, 100, 101, 103, 104, 105, 106, 107, 108, 109, 111, 113, 114, 115, 116, 117, 118, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 133, 134, 135, 136, 138, 139, 141, 144, 145, 146, 147, 148, 149, 150, 151, 153, 154, 155, 156, 157, 158, 159, 160, 163, 164, 167, 168, 169, 170, 171, 172, 175, 176, 178, 179, 180, 182, 183, 184, 185, 188, 189, 190, 191, 192, 193, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 206, 208, 210, 211, 212, 213, 216, 217, 218, 219, 220, 222, 224, 225, 227, 228, 229, 230, 231, 232, 233, 234, 236, 237, 239, 241, 242, 244, 245, 247, 248, 249, 250, 252, 253, 255, 256, 257, 258, 259, 260, 261, 262, 265, 267, 268, 269, 270, 271, 272, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 285, 286, 287, 288, 291, 292, 294, 296, 297, 298, 299, 300, 301, 303, 304, 306, 307, 308, 309, 310, 311, 312, 313, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 332, 333, 335, 336, 337, 338, 339, 340, 342, 343, 344, 345, 346, 347, 348, 349, 351, 352, 354, 355, 356, 357, 358, 359, 360, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 374, 375, 377, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 393, 395, 396, 397, 398, 399, 400, 401, 403, 404, 405, 406, 407, 408, 409, 411, 412, 413, 414, 415, 417, 418, 419], "outer_test_idx": [7, 16, 20, 29, 51, 59, 60, 69, 70, 74, 78, 79, 82, 83, 84, 87, 91, 93, 97, 102, 110, 112, 119, 120, 121, 132, 137, 140, 142, 143, 152, 161, 162, 165, 166, 173, 174, 177, 181, 186, 187, 194, 205, 207, 209, 214, 215, 221, 223, 226, 235, 238, 240, 243, 246, 251, 254, 263, 264, 266, 273, 284, 289, 290, 293, 295, 302, 305, 314, 315, 331, 334, 341, 350, 353, 361, 373, 376, 378, 392, 394, 402, 410, 416]} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_0/test_metrics.json b/models/abl_full/s2_mole/seed42/outer_fold_0/test_metrics.json new file mode 100644 index 0000000..204c7af --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_0/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 83, + "mse": 1.5121090187413508, + "rmse": 1.2296784208651264, + "mae": 0.5759273265230368, + "r2": 0.07455206011479953 + }, + "delivery": { + "n_samples": 58, + "mse": 0.5121668199501789, + "rmse": 0.7156583122902849, + "mae": 0.5583267274216331, + "r2": 0.34900506569110146 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7857142857142857, + "precision": 0.7385579937304074, + "recall": 0.7712045616535994, + "f1": 0.7493368700265252 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7380952380952381, + "precision": 0.6985851726089417, + "recall": 0.7376190476190475, + "f1": 0.7018460466736327 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9827586206896551, + "precision": 0.9912280701754386, + "recall": 0.75, + "f1": 0.8289085545722714 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.23962038921814616, + "js_divergence": 0.043880608373387 + } +} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_1/best_params.json b/models/abl_full/s2_mole/seed42/outer_fold_1/best_params.json new file mode 100644 index 0000000..b8d0ca3 --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_1/best_params.json @@ -0,0 +1,12 @@ +{ + "dropout": 0.23085562232445592, + "lr": 0.0005227270589225132, + "weight_decay": 0.00461701039547356, + "backbone_lr_ratio": 0.12087164274488635, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_1/epoch_mean.json b/models/abl_full/s2_mole/seed42/outer_fold_1/epoch_mean.json new file mode 100644 index 0000000..89e65d1 --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_1/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 7} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_1/history.json b/models/abl_full/s2_mole/seed42/outer_fold_1/history.json new file mode 100644 index 0000000..11a9a68 --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_1/history.json @@ -0,0 +1,153 @@ +{ + "train": [ + { + "loss": 5.814497735765245, + "loss_size": 1.2861030283901427, + "loss_pdi": 0.6990253494845496, + "loss_ee": 1.0696445736620162, + "loss_delivery": 0.9288370394044452, + "loss_biodist": 1.1624311672316656, + "loss_toxic": 0.6764447854624854 + }, + { + "loss": 5.153584652476841, + "loss_size": 1.1885431615842714, + "loss_pdi": 0.6575832068920135, + "loss_ee": 0.9918757246600257, + "loss_delivery": 0.8553148557742437, + "loss_biodist": 0.946597245004442, + "loss_toxic": 0.5311296731233597 + }, + { + "loss": 4.5379899342854815, + "loss_size": 1.0691165203849475, + "loss_pdi": 0.6316078570153978, + "loss_ee": 0.9180723196930356, + "loss_delivery": 0.8008341118693352, + "loss_biodist": 0.7005721744563844, + "loss_toxic": 0.4374901039732827 + }, + { + "loss": 4.146529210938348, + "loss_size": 1.1412814863853984, + "loss_pdi": 0.5906112078163359, + "loss_ee": 0.8849371208084954, + "loss_delivery": 0.7124962823258506, + "loss_biodist": 0.5163870255152384, + "loss_toxic": 0.31786360550257897 + }, + { + "loss": 3.7882408963309393, + "loss_size": 0.9951134100556374, + "loss_pdi": 0.562793321079678, + "loss_ee": 0.8465175264411502, + "loss_delivery": 0.7500328115291066, + "loss_biodist": 0.42774540599849487, + "loss_toxic": 0.2197377069128884 + }, + { + "loss": 3.6860360172059803, + "loss_size": 1.06570761402448, + "loss_pdi": 0.5500496907366647, + "loss_ee": 0.7847420573234558, + "loss_delivery": 0.700415947371059, + "loss_biodist": 0.39125806507137084, + "loss_toxic": 0.20548011838561958 + }, + { + "loss": 3.7218647135628595, + "loss_size": 1.0574430566695001, + "loss_pdi": 0.5575385275814269, + "loss_ee": 0.8225515385468801, + "loss_delivery": 0.7297804926832517, + "loss_biodist": 0.3792920708656311, + "loss_toxic": 0.1885651122364733 + } + ], + "val": [ + { + "loss": 3.7612167994181314, + "loss_size": 0.4594898638315499, + "loss_pdi": 0.6784836252530416, + "loss_ee": 1.0281402269999187, + "loss_delivery": 0.18942149720775583, + "loss_biodist": 0.8969541788101196, + "loss_toxic": 0.5171752174695333, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.696969696969697, + "acc_toxic": 1.0 + }, + { + "loss": 3.244231383005778, + "loss_size": 0.44642627735932666, + "loss_pdi": 0.6608956257502238, + "loss_ee": 0.942749559879303, + "loss_delivery": 0.19667230049769083, + "loss_biodist": 0.6659363508224487, + "loss_toxic": 0.3394078314304352, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.696969696969697, + "acc_toxic": 1.0 + }, + { + "loss": 2.9519399801890054, + "loss_size": 0.43347235520680744, + "loss_pdi": 0.6546946366628011, + "loss_ee": 0.8717599312464396, + "loss_delivery": 0.24702751139799753, + "loss_biodist": 0.5120902756849924, + "loss_toxic": 0.238857368628184, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.696969696969697, + "acc_toxic": 1.0 + }, + { + "loss": 2.971536715825399, + "loss_size": 0.5480305254459381, + "loss_pdi": 0.6065192123254141, + "loss_ee": 0.7257518072923025, + "loss_delivery": 0.6682612498601278, + "loss_biodist": 0.36282363533973694, + "loss_toxic": 0.06396338219443957, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.696969696969697, + "acc_toxic": 1.0 + }, + { + "loss": 2.949314514795939, + "loss_size": 0.5253813664118449, + "loss_pdi": 0.6070131361484528, + "loss_ee": 0.6531171997388204, + "loss_delivery": 0.7617304722468058, + "loss_biodist": 0.3587235013643901, + "loss_toxic": 0.04745769873261452, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.696969696969697, + "acc_toxic": 1.0 + }, + { + "loss": 2.7016586462656655, + "loss_size": 0.5042988359928131, + "loss_pdi": 0.610467384258906, + "loss_ee": 0.654820571343104, + "loss_delivery": 0.555947611729304, + "loss_biodist": 0.34326785306135815, + "loss_toxic": 0.03457134527464708, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.7272727272727273, + "acc_toxic": 1.0 + }, + { + "loss": 2.7002946535746255, + "loss_size": 0.49834399422009784, + "loss_pdi": 0.6154685219128927, + "loss_ee": 0.6635051965713501, + "loss_delivery": 0.5405933658281962, + "loss_biodist": 0.34967927634716034, + "loss_toxic": 0.03491388882199923, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.696969696969697, + "acc_toxic": 1.0 + } + ] +} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_1/splits.json b/models/abl_full/s2_mole/seed42/outer_fold_1/splits.json new file mode 100644 index 0000000..1773fe5 --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_1/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [0, 3, 5, 6, 7, 10, 11, 12, 13, 15, 16, 18, 19, 20, 21, 24, 26, 27, 29, 30, 31, 32, 33, 35, 36, 38, 39, 40, 42, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 65, 66, 67, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 97, 98, 99, 100, 101, 102, 103, 105, 106, 107, 108, 109, 110, 112, 114, 115, 118, 119, 120, 121, 123, 124, 125, 126, 128, 129, 130, 131, 132, 136, 137, 140, 141, 142, 143, 144, 145, 146, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 161, 162, 164, 165, 166, 167, 168, 169, 171, 172, 173, 174, 175, 176, 177, 178, 179, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 194, 195, 196, 197, 198, 199, 201, 202, 204, 205, 206, 207, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 225, 226, 228, 229, 231, 232, 233, 234, 235, 236, 237, 238, 240, 242, 243, 244, 246, 247, 248, 250, 251, 252, 254, 255, 256, 258, 259, 260, 261, 262, 263, 264, 266, 268, 269, 270, 271, 273, 274, 275, 276, 277, 278, 279, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 295, 296, 297, 300, 301, 302, 303, 304, 305, 306, 307, 308, 310, 311, 313, 314, 315, 316, 317, 318, 319, 321, 322, 323, 324, 325, 327, 328, 329, 330, 331, 333, 334, 335, 337, 338, 339, 340, 341, 342, 343, 344, 345, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 361, 362, 364, 365, 366, 367, 368, 369, 373, 374, 375, 376, 378, 379, 380, 381, 382, 384, 387, 388, 389, 391, 392, 393, 394, 396, 397, 398, 399, 402, 403, 404, 405, 406, 408, 409, 410, 411, 412, 413, 414, 416, 417, 419], "outer_test_idx": [1, 2, 4, 8, 9, 14, 17, 22, 23, 25, 28, 34, 37, 41, 43, 45, 63, 64, 68, 85, 96, 104, 111, 113, 116, 117, 122, 127, 133, 134, 135, 138, 139, 147, 160, 163, 170, 180, 191, 192, 193, 200, 203, 208, 224, 227, 230, 239, 241, 245, 249, 253, 257, 265, 267, 272, 280, 294, 298, 299, 309, 312, 320, 326, 332, 336, 346, 347, 360, 363, 370, 371, 372, 377, 383, 385, 386, 390, 395, 400, 401, 407, 415, 418]} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_1/test_metrics.json b/models/abl_full/s2_mole/seed42/outer_fold_1/test_metrics.json new file mode 100644 index 0000000..8194746 --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_1/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.3284213709926875, + "rmse": 0.5730805972921151, + "mae": 0.45920564744266723, + "r2": -0.002882105767083143 + }, + "delivery": { + "n_samples": 61, + "mse": 1.0560295226310608, + "rmse": 1.0276329707785075, + "mae": 0.6466741458070083, + "r2": 0.2006617794084804 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6666666666666666, + "precision": 0.5886939571150097, + "recall": 0.6031746031746031, + "f1": 0.5916666666666667 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5833333333333334, + "precision": 0.5271672771672772, + "recall": 0.533918462489891, + "f1": 0.513073133762789 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.4648032547075186, + "js_divergence": 0.11675330313943091 + } +} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_2/best_params.json b/models/abl_full/s2_mole/seed42/outer_fold_2/best_params.json new file mode 100644 index 0000000..bb85933 --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_2/best_params.json @@ -0,0 +1,12 @@ +{ + "dropout": 0.11129874930397193, + "lr": 0.000993106647576575, + "weight_decay": 0.0015528212338926973, + "backbone_lr_ratio": 0.19356048624251188, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_2/epoch_mean.json b/models/abl_full/s2_mole/seed42/outer_fold_2/epoch_mean.json new file mode 100644 index 0000000..17fb17e --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_2/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 10} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_2/history.json b/models/abl_full/s2_mole/seed42/outer_fold_2/history.json new file mode 100644 index 0000000..3b2162c --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_2/history.json @@ -0,0 +1,216 @@ +{ + "train": [ + { + "loss": 4.949816968705919, + "loss_size": 0.7250039892064201, + "loss_pdi": 0.6515774230162302, + "loss_ee": 1.034024437268575, + "loss_delivery": 0.8304822180006239, + "loss_biodist": 1.0945891241232555, + "loss_toxic": 0.625051349401474 + }, + { + "loss": 4.030560255050659, + "loss_size": 0.7653147735529475, + "loss_pdi": 0.5911111169391208, + "loss_ee": 0.8889327910211351, + "loss_delivery": 0.8114540154735247, + "loss_biodist": 0.5877454380194346, + "loss_toxic": 0.4012700815995534 + }, + { + "loss": 3.4370746745003595, + "loss_size": 0.7095198250479169, + "loss_pdi": 0.5523345751894845, + "loss_ee": 0.8699113329251608, + "loss_delivery": 0.7007193250788583, + "loss_biodist": 0.34551991687880623, + "loss_toxic": 0.2726496176587211 + }, + { + "loss": 3.256797605090671, + "loss_size": 0.6292490392095513, + "loss_pdi": 0.5541596611340841, + "loss_ee": 0.8518470062149895, + "loss_delivery": 0.6860647855533494, + "loss_biodist": 0.3054388165473938, + "loss_toxic": 0.2500717712359296 + }, + { + "loss": 2.922978083292643, + "loss_size": 0.6592319077915616, + "loss_pdi": 0.4948728531599045, + "loss_ee": 0.7609278824594286, + "loss_delivery": 0.6493541424473127, + "loss_biodist": 0.2407314996752474, + "loss_toxic": 0.13528130208659503 + }, + { + "loss": 2.503936860296461, + "loss_size": 0.42642495450046325, + "loss_pdi": 0.48528778553009033, + "loss_ee": 0.7667093839910295, + "loss_delivery": 0.5656192344095972, + "loss_biodist": 0.18141375647650826, + "loss_toxic": 0.09636203427281645 + }, + { + "loss": 2.3560825188954673, + "loss_size": 0.43924198175470036, + "loss_pdi": 0.4449809193611145, + "loss_ee": 0.7099536375866996, + "loss_delivery": 0.5192352810667621, + "loss_biodist": 0.16359280546506247, + "loss_toxic": 0.0944044955023047 + }, + { + "loss": 2.3559987677468195, + "loss_size": 0.4138515488141113, + "loss_pdi": 0.47462597903278136, + "loss_ee": 0.6494933399889204, + "loss_delivery": 0.55873135311736, + "loss_biodist": 0.18862591476904023, + "loss_toxic": 0.0875174353002674 + }, + { + "loss": 2.1793322298261852, + "loss_size": 0.35637974159585106, + "loss_pdi": 0.44880036844147575, + "loss_ee": 0.6503889312346777, + "loss_delivery": 0.5077870993150605, + "loss_biodist": 0.16400393636690247, + "loss_toxic": 0.06829726111350788 + }, + { + "loss": 2.1951819724506803, + "loss_size": 0.3660747582713763, + "loss_pdi": 0.42095059487554765, + "loss_ee": 0.6363236589564217, + "loss_delivery": 0.5179207846522331, + "loss_biodist": 0.1654499489814043, + "loss_toxic": 0.10260950680822134 + } + ], + "val": [ + { + "loss": 7.470086177190145, + "loss_size": 3.3436052997907004, + "loss_pdi": 0.6940620342890421, + "loss_ee": 1.032279133796692, + "loss_delivery": 1.4374334216117859, + "loss_biodist": 0.6329567631085714, + "loss_toxic": 0.36738624175389606, + "acc_pdi": 0.3939393939393939, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.95 + }, + { + "loss": 6.868675231933594, + "loss_size": 3.3082821865876517, + "loss_pdi": 0.7149907151858012, + "loss_ee": 0.980406661828359, + "loss_delivery": 1.3947720626990001, + "loss_biodist": 0.30634867151578266, + "loss_toxic": 0.20873196919759116, + "acc_pdi": 0.5454545454545454, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.95 + }, + { + "loss": 6.57732089360555, + "loss_size": 3.3375285466512046, + "loss_pdi": 0.6716682513554891, + "loss_ee": 0.9775587320327759, + "loss_delivery": 1.2827350497245789, + "loss_biodist": 0.21934483448664346, + "loss_toxic": 0.1425950825214386, + "acc_pdi": 0.5454545454545454, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.95 + }, + { + "loss": 6.636128902435303, + "loss_size": 3.3263390262921653, + "loss_pdi": 0.7673717538515726, + "loss_ee": 1.0473139484723408, + "loss_delivery": 1.3343796928723652, + "loss_biodist": 0.1280744175116221, + "loss_toxic": 0.10122497131427129, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.95 + }, + { + "loss": 6.141360918680827, + "loss_size": 2.929147313038508, + "loss_pdi": 0.6981973846753439, + "loss_ee": 1.1199428240458171, + "loss_delivery": 1.27666437625885, + "loss_biodist": 0.12178416053454082, + "loss_toxic": 0.06837374344468117, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.95 + }, + { + "loss": 5.823832114537557, + "loss_size": 2.6620603601137796, + "loss_pdi": 0.6536334355672201, + "loss_ee": 1.102464238802592, + "loss_delivery": 1.2998173236846924, + "loss_biodist": 0.11245918273925781, + "loss_toxic": 0.0649159774184227, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.6363636363636364, + "acc_toxic": 0.95 + }, + { + "loss": 5.9035530885060625, + "loss_size": 2.7045720914999642, + "loss_pdi": 0.6657395958900452, + "loss_ee": 1.1599463025728862, + "loss_delivery": 1.2662039597829182, + "loss_biodist": 0.11702268570661545, + "loss_toxic": 0.06584630409876506, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.95 + }, + { + "loss": 5.937086423238118, + "loss_size": 2.7061472733815513, + "loss_pdi": 0.6816494961579641, + "loss_ee": 1.1641804575920105, + "loss_delivery": 1.2764110167821248, + "loss_biodist": 0.12533247222503027, + "loss_toxic": 0.06109336391091347, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.95 + }, + { + "loss": 5.867360194524129, + "loss_size": 2.663755605618159, + "loss_pdi": 0.676131933927536, + "loss_ee": 1.1532418131828308, + "loss_delivery": 1.2642570336659749, + "loss_biodist": 0.1211617464820544, + "loss_toxic": 0.06515560174981753, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.95 + }, + { + "loss": 5.906121492385864, + "loss_size": 2.7017226914564767, + "loss_pdi": 0.6825120449066162, + "loss_ee": 1.160526434580485, + "loss_delivery": 1.2439844608306885, + "loss_biodist": 0.12497460593779881, + "loss_toxic": 0.06895889714360237, + "acc_pdi": 0.7878787878787878, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.95 + } + ] +} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_2/splits.json b/models/abl_full/s2_mole/seed42/outer_fold_2/splits.json new file mode 100644 index 0000000..14df306 --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_2/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 36, 37, 38, 39, 40, 41, 43, 44, 45, 47, 48, 49, 50, 51, 55, 56, 57, 58, 59, 60, 61, 63, 64, 65, 68, 69, 70, 72, 74, 76, 78, 79, 80, 82, 83, 84, 85, 86, 87, 88, 91, 93, 94, 95, 96, 97, 100, 101, 102, 103, 104, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 142, 143, 145, 147, 148, 149, 150, 151, 152, 153, 156, 157, 158, 159, 160, 161, 162, 163, 165, 166, 167, 168, 170, 171, 172, 173, 174, 175, 177, 179, 180, 181, 183, 184, 185, 186, 187, 188, 191, 192, 193, 194, 196, 197, 199, 200, 202, 203, 204, 205, 207, 208, 209, 210, 211, 212, 213, 214, 215, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 232, 233, 235, 236, 237, 238, 239, 240, 241, 242, 243, 245, 246, 249, 250, 251, 252, 253, 254, 255, 256, 257, 259, 260, 261, 263, 264, 265, 266, 267, 268, 269, 271, 272, 273, 274, 275, 276, 280, 281, 282, 283, 284, 285, 287, 289, 290, 293, 294, 295, 296, 297, 298, 299, 300, 302, 303, 305, 307, 308, 309, 310, 311, 312, 313, 314, 315, 317, 318, 320, 321, 322, 323, 324, 326, 327, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 358, 360, 361, 363, 364, 365, 366, 368, 369, 370, 371, 372, 373, 376, 377, 378, 379, 380, 381, 382, 383, 385, 386, 387, 388, 389, 390, 392, 394, 395, 398, 399, 400, 401, 402, 404, 405, 407, 408, 409, 410, 412, 413, 414, 415, 416, 417, 418], "outer_test_idx": [10, 12, 15, 31, 35, 42, 46, 52, 53, 54, 62, 66, 67, 71, 73, 75, 77, 81, 89, 90, 92, 98, 99, 105, 123, 124, 141, 144, 146, 154, 155, 164, 169, 176, 178, 182, 189, 190, 195, 198, 201, 206, 216, 217, 218, 219, 231, 234, 244, 247, 248, 258, 262, 270, 277, 278, 279, 286, 288, 291, 292, 301, 304, 306, 316, 319, 325, 328, 343, 357, 359, 362, 367, 374, 375, 384, 391, 393, 396, 397, 403, 406, 411, 419]} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_2/test_metrics.json b/models/abl_full/s2_mole/seed42/outer_fold_2/test_metrics.json new file mode 100644 index 0000000..97921d4 --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_2/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.395384611648953, + "rmse": 0.6287961606506142, + "mae": 0.4870017299295536, + "r2": 0.219122456317145 + }, + "delivery": { + "n_samples": 60, + "mse": 0.692732678583736, + "rmse": 0.8323056401249099, + "mae": 0.5703436779246355, + "r2": 0.07785273767480438 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7023809523809523, + "precision": 0.6503347534996957, + "recall": 0.6810850439882699, + "f1": 0.6554552912223134 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6547619047619048, + "precision": 0.6078622482131254, + "recall": 0.6720354808590102, + "f1": 0.6212949538882847 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.179209086609629, + "js_divergence": 0.0456761395398062 + } +} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_3/best_params.json b/models/abl_full/s2_mole/seed42/outer_fold_3/best_params.json new file mode 100644 index 0000000..48538f8 --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_3/best_params.json @@ -0,0 +1,12 @@ +{ + "dropout": 0.19585998114210643, + "lr": 0.0008949481903172137, + "weight_decay": 0.0848478556523433, + "backbone_lr_ratio": 0.7944603821606866, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_3/epoch_mean.json b/models/abl_full/s2_mole/seed42/outer_fold_3/epoch_mean.json new file mode 100644 index 0000000..735a6b9 --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_3/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 11} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_3/history.json b/models/abl_full/s2_mole/seed42/outer_fold_3/history.json new file mode 100644 index 0000000..434ca46 --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_3/history.json @@ -0,0 +1,237 @@ +{ + "train": [ + { + "loss": 4.907192256715563, + "loss_size": 0.42668238944477505, + "loss_pdi": 0.6802443861961365, + "loss_ee": 1.0871289637353685, + "loss_delivery": 0.9925478398799896, + "loss_biodist": 1.124436742729611, + "loss_toxic": 0.6067684392134348 + }, + { + "loss": 3.9632872210608587, + "loss_size": 0.45460548334651524, + "loss_pdi": 0.634620875120163, + "loss_ee": 0.9712704949908786, + "loss_delivery": 0.8261574606100718, + "loss_biodist": 0.7236964586708281, + "loss_toxic": 0.372286732825968 + }, + { + "loss": 3.4079616467158, + "loss_size": 0.4338843632075522, + "loss_pdi": 0.5915416346655952, + "loss_ee": 0.915321860048506, + "loss_delivery": 0.7535964097413752, + "loss_biodist": 0.47949627869659, + "loss_toxic": 0.2550220257706112 + }, + { + "loss": 3.529743512471517, + "loss_size": 0.5470549580123689, + "loss_pdi": 0.5998486512237124, + "loss_ee": 0.9072406954235501, + "loss_delivery": 0.8790290405352911, + "loss_biodist": 0.42277195925513905, + "loss_toxic": 0.2022996568121016 + }, + { + "loss": 3.153548796971639, + "loss_size": 0.4619236108329561, + "loss_pdi": 0.5547024988465838, + "loss_ee": 0.8648580511411031, + "loss_delivery": 0.8316900713576211, + "loss_biodist": 0.30215797076622647, + "loss_toxic": 0.16683673268804947 + }, + { + "loss": 3.006254209412469, + "loss_size": 0.45004433890183765, + "loss_pdi": 0.5253052877055274, + "loss_ee": 0.8526065680715773, + "loss_delivery": 0.7567387877239121, + "loss_biodist": 0.2727365580697854, + "loss_toxic": 0.1780139264754123 + }, + { + "loss": 2.6917024850845337, + "loss_size": 0.4284028551644749, + "loss_pdi": 0.4961979471974903, + "loss_ee": 0.8068405654695299, + "loss_delivery": 0.67920307483938, + "loss_biodist": 0.20536433988147312, + "loss_toxic": 0.1052106049286926 + }, + { + "loss": 2.6041518449783325, + "loss_size": 0.37318388041522765, + "loss_pdi": 0.4925161235862308, + "loss_ee": 0.7617953055434756, + "loss_delivery": 0.6340341303083632, + "loss_biodist": 0.24038264238172108, + "loss_toxic": 0.13249483223383626 + }, + { + "loss": 2.466211107042101, + "loss_size": 0.3615158614185121, + "loss_pdi": 0.5065714882479774, + "loss_ee": 0.7162391990423203, + "loss_delivery": 0.6294859097235732, + "loss_biodist": 0.17513174811999002, + "loss_toxic": 0.1078372763004154 + }, + { + "loss": 2.294809182484945, + "loss_size": 0.3293912145826552, + "loss_pdi": 0.48184214449591106, + "loss_ee": 0.6876992020342085, + "loss_delivery": 0.6090891311566035, + "loss_biodist": 0.15249460852808422, + "loss_toxic": 0.06555601530190971 + }, + { + "loss": 2.243659734725952, + "loss_size": 0.331886964953608, + "loss_pdi": 0.4913172754976485, + "loss_ee": 0.6733519534269968, + "loss_delivery": 0.5500031676557329, + "loss_biodist": 0.15894580963585112, + "loss_toxic": 0.06820586101255482 + } + ], + "val": [ + { + "loss": 5.586382110913594, + "loss_size": 2.3582379817962646, + "loss_pdi": 0.6279834707578024, + "loss_ee": 0.9819174011548361, + "loss_delivery": 0.7756642053524653, + "loss_biodist": 0.5739627679189047, + "loss_toxic": 0.27646539608637494, + "acc_pdi": 0.8181818181818182, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.9565217391304348 + }, + { + "loss": 4.929585297902425, + "loss_size": 2.278209318717321, + "loss_pdi": 0.5694902539253235, + "loss_ee": 0.7597478330135345, + "loss_delivery": 0.826200395822525, + "loss_biodist": 0.2612158755461375, + "loss_toxic": 0.2287526528040568, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.696969696969697, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 4.84937047958374, + "loss_size": 2.259629930059115, + "loss_pdi": 0.6305497288703918, + "loss_ee": 0.7094869414965311, + "loss_delivery": 0.835543672243754, + "loss_biodist": 0.192092627286911, + "loss_toxic": 0.20736992359161377, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.6666666666666666, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 5.243381023406982, + "loss_size": 2.042687470714251, + "loss_pdi": 0.6461743315060934, + "loss_ee": 0.7939094106356303, + "loss_delivery": 0.8636383165915807, + "loss_biodist": 0.21153046687444052, + "loss_toxic": 0.6735498507817587, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.6363636363636364, + "acc_toxic": 0.782608695652174 + }, + { + "loss": 4.794024427731832, + "loss_size": 2.5530659407377243, + "loss_pdi": 0.47391365468502045, + "loss_ee": 0.7832625309626261, + "loss_delivery": 0.6480557272831599, + "loss_biodist": 0.21558143695195517, + "loss_toxic": 0.09970808525880177, + "acc_pdi": 0.8181818181818182, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 4.382653474807739, + "loss_size": 2.075940708319346, + "loss_pdi": 0.5197472075621287, + "loss_ee": 0.888870487610499, + "loss_delivery": 0.5837112665176392, + "loss_biodist": 0.1355785479148229, + "loss_toxic": 0.17190168797969818, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.48484848484848486, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 4.007976611455281, + "loss_size": 1.7206853528817494, + "loss_pdi": 0.5513814489046732, + "loss_ee": 0.7508460481961569, + "loss_delivery": 0.6324484646320343, + "loss_biodist": 0.13428052266438803, + "loss_toxic": 0.21318833033243814, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.5151515151515151, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 4.218830386797587, + "loss_size": 1.922361637155215, + "loss_pdi": 0.53309366106987, + "loss_ee": 0.8696597814559937, + "loss_delivery": 0.5403677026430765, + "loss_biodist": 0.15033350884914398, + "loss_toxic": 0.19822034239768982, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 3.9281427462895713, + "loss_size": 1.9172342518965404, + "loss_pdi": 0.47034143408139545, + "loss_ee": 0.7674199243386587, + "loss_delivery": 0.533902108669281, + "loss_biodist": 0.09907602022091548, + "loss_toxic": 0.12868722279866537, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 3.9501442909240723, + "loss_size": 1.90011170009772, + "loss_pdi": 0.4726162652174632, + "loss_ee": 0.811152994632721, + "loss_delivery": 0.4848788032929103, + "loss_biodist": 0.10525307059288025, + "loss_toxic": 0.16470323503017426, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 3.960719267527262, + "loss_size": 1.9160331189632416, + "loss_pdi": 0.48155856132507324, + "loss_ee": 0.8100563883781433, + "loss_delivery": 0.4847661703824997, + "loss_biodist": 0.10654482742150624, + "loss_toxic": 0.15084098279476166, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.8695652173913043 + } + ] +} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_3/splits.json b/models/abl_full/s2_mole/seed42/outer_fold_3/splits.json new file mode 100644 index 0000000..7a2b13e --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_3/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [0, 1, 2, 4, 5, 7, 8, 9, 10, 11, 12, 14, 15, 16, 17, 20, 21, 22, 23, 24, 25, 27, 28, 29, 30, 31, 32, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 45, 46, 47, 51, 52, 53, 54, 55, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 73, 74, 75, 76, 77, 78, 79, 81, 82, 83, 84, 85, 86, 87, 89, 90, 91, 92, 93, 95, 96, 97, 98, 99, 100, 102, 104, 105, 106, 109, 110, 111, 112, 113, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 127, 129, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 146, 147, 148, 150, 151, 152, 153, 154, 155, 156, 158, 159, 160, 161, 162, 163, 164, 165, 166, 168, 169, 170, 172, 173, 174, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 189, 190, 191, 192, 193, 194, 195, 198, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 212, 213, 214, 215, 216, 217, 218, 219, 221, 223, 224, 226, 227, 228, 230, 231, 233, 234, 235, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 251, 252, 253, 254, 255, 256, 257, 258, 259, 262, 263, 264, 265, 266, 267, 268, 269, 270, 272, 273, 274, 275, 276, 277, 278, 279, 280, 282, 284, 286, 288, 289, 290, 291, 292, 293, 294, 295, 297, 298, 299, 301, 302, 304, 305, 306, 307, 308, 309, 310, 312, 314, 315, 316, 317, 319, 320, 321, 325, 326, 327, 328, 331, 332, 334, 336, 340, 341, 343, 344, 346, 347, 348, 350, 352, 353, 357, 359, 360, 361, 362, 363, 364, 366, 367, 368, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 382, 383, 384, 385, 386, 387, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 400, 401, 402, 403, 404, 406, 407, 410, 411, 412, 414, 415, 416, 418, 419], "outer_test_idx": [3, 6, 13, 18, 19, 26, 33, 44, 48, 49, 50, 56, 57, 72, 80, 88, 94, 101, 103, 107, 108, 114, 126, 128, 130, 145, 149, 157, 167, 171, 175, 188, 196, 197, 199, 211, 220, 222, 225, 229, 232, 236, 250, 260, 261, 271, 281, 283, 285, 287, 296, 300, 303, 311, 313, 318, 322, 323, 324, 329, 330, 333, 335, 337, 338, 339, 342, 345, 349, 351, 354, 355, 356, 358, 365, 369, 381, 388, 399, 405, 408, 409, 413, 417]} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_3/test_metrics.json b/models/abl_full/s2_mole/seed42/outer_fold_3/test_metrics.json new file mode 100644 index 0000000..c82dc3d --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_3/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 83, + "mse": 1.7351497225620842, + "rmse": 1.317250819913233, + "mae": 0.577601661647569, + "r2": 0.018586091772742508 + }, + "delivery": { + "n_samples": 59, + "mse": 0.6791751469481081, + "rmse": 0.8241208327351688, + "mae": 0.5204403357959148, + "r2": 0.3397610637519505 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7142857142857143, + "precision": 0.6375, + "recall": 0.6451612903225806, + "f1": 0.6407697790449038 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6309523809523809, + "precision": 0.5632266806539034, + "recall": 0.5736299648064355, + "f1": 0.5568911622620062 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9166666666666666, + "precision": 0.6875, + "recall": 0.956140350877193, + "f1": 0.749791492910759 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.17262215641467948, + "js_divergence": 0.04320988831477907 + } +} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_4/best_params.json b/models/abl_full/s2_mole/seed42/outer_fold_4/best_params.json new file mode 100644 index 0000000..64a4555 --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_4/best_params.json @@ -0,0 +1,12 @@ +{ + "dropout": 0.22523077425267193, + "lr": 0.0007489877504872848, + "weight_decay": 1.4840953126290499e-05, + "backbone_lr_ratio": 0.07616131024702885, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_4/epoch_mean.json b/models/abl_full/s2_mole/seed42/outer_fold_4/epoch_mean.json new file mode 100644 index 0000000..0ac5b47 --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_4/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 13} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_4/history.json b/models/abl_full/s2_mole/seed42/outer_fold_4/history.json new file mode 100644 index 0000000..699c28f --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_4/history.json @@ -0,0 +1,279 @@ +{ + "train": [ + { + "loss": 5.601248900095622, + "loss_size": 1.1405156039529376, + "loss_pdi": 0.6812781227959527, + "loss_ee": 1.0623916586240132, + "loss_delivery": 0.9658779551585516, + "loss_biodist": 1.156044711669286, + "loss_toxic": 0.6033158418205049 + }, + { + "loss": 4.7204041083653765, + "loss_size": 1.0748717967006896, + "loss_pdi": 0.655184633202023, + "loss_ee": 0.9847381313641866, + "loss_delivery": 0.7530446474750837, + "loss_biodist": 0.8042493528789945, + "loss_toxic": 0.4675862507687675 + }, + { + "loss": 4.348466131422255, + "loss_size": 1.0511629540059302, + "loss_pdi": 0.612076097064548, + "loss_ee": 0.9314051469167074, + "loss_delivery": 0.8202763886915313, + "loss_biodist": 0.5366683635446761, + "loss_toxic": 0.42014338158898884 + }, + { + "loss": 3.898900628089905, + "loss_size": 1.0103039965033531, + "loss_pdi": 0.5797004203001658, + "loss_ee": 0.865445014503267, + "loss_delivery": 0.7599279251363542, + "loss_biodist": 0.43361951659123105, + "loss_toxic": 0.2724840607908037 + }, + { + "loss": 3.528401917881436, + "loss_size": 0.9298759135935042, + "loss_pdi": 0.5419453630844752, + "loss_ee": 0.8654191195964813, + "loss_delivery": 0.6367430380649037, + "loss_biodist": 0.3877827028433482, + "loss_toxic": 0.18729782145884302 + }, + { + "loss": 3.2993942499160767, + "loss_size": 0.8436437853508525, + "loss_pdi": 0.5482367674509684, + "loss_ee": 0.8198477592733171, + "loss_delivery": 0.6481610310988294, + "loss_biodist": 0.32835982657141155, + "loss_toxic": 0.1312161408778694 + }, + { + "loss": 3.1544810401068792, + "loss_size": 0.842688586976793, + "loss_pdi": 0.5108132974969016, + "loss_ee": 0.7503058678574033, + "loss_delivery": 0.6542684841487143, + "loss_biodist": 0.3094532862305641, + "loss_toxic": 0.1052164782045616 + }, + { + "loss": 3.062612328264448, + "loss_size": 0.716158828801579, + "loss_pdi": 0.5271133780479431, + "loss_ee": 0.7655319737063514, + "loss_delivery": 0.6855515506532457, + "loss_biodist": 0.2818622845742438, + "loss_toxic": 0.10757030644971463 + }, + { + "loss": 3.0127061472998724, + "loss_size": 0.7125719512502352, + "loss_pdi": 0.5221862892309824, + "loss_ee": 0.7723172770606147, + "loss_delivery": 0.6382502441604933, + "loss_biodist": 0.2745503592822287, + "loss_toxic": 0.11453904211521149 + }, + { + "loss": 2.864455262819926, + "loss_size": 0.6301954736312231, + "loss_pdi": 0.5172365026341544, + "loss_ee": 0.7678910924328698, + "loss_delivery": 0.6224940261907048, + "loss_biodist": 0.25988349070151645, + "loss_toxic": 0.08878863318305877 + }, + { + "loss": 3.0430398318502636, + "loss_size": 0.8107057478692796, + "loss_pdi": 0.5051789995696809, + "loss_ee": 0.7711051371362474, + "loss_delivery": 0.6184873452617062, + "loss_biodist": 0.2630308171113332, + "loss_toxic": 0.09840117136223449 + }, + { + "loss": 2.9058522250917225, + "loss_size": 0.6377110481262207, + "loss_pdi": 0.5315848870409859, + "loss_ee": 0.7531339327494303, + "loss_delivery": 0.6412442355520196, + "loss_biodist": 0.24950542052586874, + "loss_toxic": 0.11562347412109375 + }, + { + "loss": 2.880252765284644, + "loss_size": 0.6427769478824403, + "loss_pdi": 0.5011419951915741, + "loss_ee": 0.7675267060597738, + "loss_delivery": 0.6194256055686209, + "loss_biodist": 0.26933180292447406, + "loss_toxic": 0.10296267012341155 + } + ], + "val": [ + { + "loss": 4.4736870129903155, + "loss_size": 0.6189786692460378, + "loss_pdi": 0.634761651357015, + "loss_ee": 0.9558977286020914, + "loss_delivery": 0.7127848425880075, + "loss_biodist": 1.009098490079244, + "loss_toxic": 0.5586141149202982, + "acc_pdi": 0.7878787878787878, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 3.8358700275421143, + "loss_size": 0.44545479615529376, + "loss_pdi": 0.5912291804949442, + "loss_ee": 0.8513731360435486, + "loss_delivery": 0.811211391662558, + "loss_biodist": 0.6578506429990133, + "loss_toxic": 0.49709176023801166, + "acc_pdi": 0.5151515151515151, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 3.355198701222738, + "loss_size": 0.36784571905930835, + "loss_pdi": 0.5488886634508768, + "loss_ee": 0.774930735429128, + "loss_delivery": 0.6961656886851415, + "loss_biodist": 0.4778725206851959, + "loss_toxic": 0.5026083389918009, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 2.9714744885762534, + "loss_size": 0.2271790529290835, + "loss_pdi": 0.44279175500075024, + "loss_ee": 0.6834543844064077, + "loss_delivery": 0.8228580007950465, + "loss_biodist": 0.2757170448700587, + "loss_toxic": 0.5229481520752112, + "acc_pdi": 0.48484848484848486, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 2.895413120587667, + "loss_size": 0.2160253127415975, + "loss_pdi": 0.41266152014334995, + "loss_ee": 0.6763803772628307, + "loss_delivery": 0.85369140903155, + "loss_biodist": 0.16389826933542886, + "loss_toxic": 0.5726009470721086, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 2.8808323542277017, + "loss_size": 0.28274618089199066, + "loss_pdi": 0.40333599348862964, + "loss_ee": 0.6499751855929693, + "loss_delivery": 0.8145729654158155, + "loss_biodist": 0.16396349420150122, + "loss_toxic": 0.5644437211255232, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 3.050659497578939, + "loss_size": 0.37641528248786926, + "loss_pdi": 0.39257606863975525, + "loss_ee": 0.6451906468719244, + "loss_delivery": 0.8641558711727461, + "loss_biodist": 0.14331893188258013, + "loss_toxic": 0.6272348028918108, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 2.8346150318781533, + "loss_size": 0.3541935781637828, + "loss_pdi": 0.3952992707490921, + "loss_ee": 0.6617893396566311, + "loss_delivery": 0.8279366409406066, + "loss_biodist": 0.129398542145888, + "loss_toxic": 0.467301981523633, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 2.7594239314397178, + "loss_size": 0.3342180997133255, + "loss_pdi": 0.39890141288439435, + "loss_ee": 0.6397652433564266, + "loss_delivery": 0.8071154284601411, + "loss_biodist": 0.12523290266593298, + "loss_toxic": 0.45474949789543945, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 2.834681431452433, + "loss_size": 0.31826506555080414, + "loss_pdi": 0.3912368267774582, + "loss_ee": 0.6579239039371411, + "loss_delivery": 0.8480330916742483, + "loss_biodist": 0.11345629518230756, + "loss_toxic": 0.50667355209589, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 2.8501628637313843, + "loss_size": 0.31981803476810455, + "loss_pdi": 0.3901669681072235, + "loss_ee": 0.6626553374032179, + "loss_delivery": 0.8580092079937458, + "loss_biodist": 0.11090129800140858, + "loss_toxic": 0.5100756734609604, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 2.8663405179977417, + "loss_size": 0.3215045581261317, + "loss_pdi": 0.38965294261773425, + "loss_ee": 0.6564014938970407, + "loss_delivery": 0.8575774803757668, + "loss_biodist": 0.11197181170185407, + "loss_toxic": 0.5299235402295986, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 2.8603278001149497, + "loss_size": 0.30929245551427204, + "loss_pdi": 0.38745732108751935, + "loss_ee": 0.669533284381032, + "loss_delivery": 0.8605100760857264, + "loss_biodist": 0.10875449081261952, + "loss_toxic": 0.5260920530805985, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.8888888888888888 + } + ] +} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_4/splits.json b/models/abl_full/s2_mole/seed42/outer_fold_4/splits.json new file mode 100644 index 0000000..8dd0299 --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_4/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23, 25, 26, 28, 29, 31, 33, 34, 35, 37, 41, 42, 43, 44, 45, 46, 48, 49, 50, 51, 52, 53, 54, 56, 57, 59, 60, 62, 63, 64, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 77, 78, 79, 80, 81, 82, 83, 84, 85, 87, 88, 89, 90, 91, 92, 93, 94, 96, 97, 98, 99, 101, 102, 103, 104, 105, 107, 108, 110, 111, 112, 113, 114, 116, 117, 119, 120, 121, 122, 123, 124, 126, 127, 128, 130, 132, 133, 134, 135, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 149, 152, 154, 155, 157, 160, 161, 162, 163, 164, 165, 166, 167, 169, 170, 171, 173, 174, 175, 176, 177, 178, 180, 181, 182, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 203, 205, 206, 207, 208, 209, 211, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 229, 230, 231, 232, 234, 235, 236, 238, 239, 240, 241, 243, 244, 245, 246, 247, 248, 249, 250, 251, 253, 254, 257, 258, 260, 261, 262, 263, 264, 265, 266, 267, 270, 271, 272, 273, 277, 278, 279, 280, 281, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 298, 299, 300, 301, 302, 303, 304, 305, 306, 309, 311, 312, 313, 314, 315, 316, 318, 319, 320, 322, 323, 324, 325, 326, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 341, 342, 343, 345, 346, 347, 349, 350, 351, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 365, 367, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 381, 383, 384, 385, 386, 388, 390, 391, 392, 393, 394, 395, 396, 397, 399, 400, 401, 402, 403, 405, 406, 407, 408, 409, 410, 411, 413, 415, 416, 417, 418, 419], "outer_test_idx": [0, 5, 11, 21, 24, 27, 30, 32, 36, 38, 39, 40, 47, 55, 58, 61, 65, 76, 86, 95, 100, 106, 109, 115, 118, 125, 129, 131, 136, 148, 150, 151, 153, 156, 158, 159, 168, 172, 179, 183, 184, 185, 202, 204, 210, 212, 213, 228, 233, 237, 242, 252, 255, 256, 259, 268, 269, 274, 275, 276, 282, 297, 307, 308, 310, 317, 321, 327, 340, 344, 348, 352, 364, 366, 368, 379, 380, 382, 387, 389, 398, 404, 412, 414]} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/outer_fold_4/test_metrics.json b/models/abl_full/s2_mole/seed42/outer_fold_4/test_metrics.json new file mode 100644 index 0000000..82b1908 --- /dev/null +++ b/models/abl_full/s2_mole/seed42/outer_fold_4/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.5768887598177137, + "rmse": 0.7595319346924879, + "mae": 0.4391206152282566, + "r2": 0.22805118060144536 + }, + "delivery": { + "n_samples": 58, + "mse": 0.923829878979012, + "rmse": 0.9611606936298488, + "mae": 0.6288489371152788, + "r2": 0.1066717577051215 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6904761904761905, + "precision": 0.6333333333333333, + "recall": 0.6583577712609971, + "f1": 0.6379310344827587 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6904761904761905, + "precision": 0.6374458874458875, + "recall": 0.676068376068376, + "f1": 0.6504784466740988 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.8659090909090909 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.26043390803767386, + "js_divergence": 0.07162115663497268 + } +} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/strata_info.json b/models/abl_full/s2_mole/seed42/strata_info.json new file mode 100644 index 0000000..ef3be58 --- /dev/null +++ b/models/abl_full/s2_mole/seed42/strata_info.json @@ -0,0 +1,60 @@ +{ + "original_strata_counts": { + "T0|P0|E0": "5", + "T0|P0|E1": "58", + "T0|P0|E2": "169", + "T0|P1|E0": "1", + "T0|P1|E1": "17", + "T0|P1|E2": "32", + "T0|P2|E2": "3", + "T1|P0|E2": "9", + "T1|P1|E2": "5", + "TNA|P0|E0": "28", + "TNA|P0|E1": "21", + "TNA|P0|E2": "20", + "TNA|P1|E0": "29", + "TNA|P1|E1": "7", + "TNA|P1|E2": "14", + "TNA|P2|E2": "1", + "TNA|P3|E0": "1" + }, + "rare_strata": [ + "T0|P1|E0", + "T0|P2|E2", + "TNA|P2|E2", + "TNA|P3|E0" + ], + "final_strata": [ + "RARE", + "T0|P0|E0", + "T0|P0|E1", + "T0|P0|E2", + "T0|P1|E1", + "T0|P1|E2", + "T1|P0|E2", + "T1|P1|E2", + "TNA|P0|E0", + "TNA|P0|E1", + "TNA|P0|E2", + "TNA|P1|E0", + "TNA|P1|E1", + "TNA|P1|E2" + ], + "final_strata_counts": { + "RARE": "6", + "T0|P0|E0": "5", + "T0|P0|E1": "58", + "T0|P0|E2": "169", + "T0|P1|E1": "17", + "T0|P1|E2": "32", + "T1|P0|E2": "9", + "T1|P1|E2": "5", + "TNA|P0|E0": "28", + "TNA|P0|E1": "21", + "TNA|P0|E2": "20", + "TNA|P1|E0": "29", + "TNA|P1|E1": "7", + "TNA|P1|E2": "14" + }, + "n_rare_merged": "6" +} \ No newline at end of file diff --git a/models/abl_full/s2_mole/seed42/summary.json b/models/abl_full/s2_mole/seed42/summary.json new file mode 100644 index 0000000..13c433f --- /dev/null +++ b/models/abl_full/s2_mole/seed42/summary.json @@ -0,0 +1,352 @@ +{ + "fold_results": [ + { + "fold": 0, + "best_params": { + "dropout": 0.10525612628712336, + "lr": 0.0008524636948952184, + "weight_decay": 0.00045548267300368544, + "backbone_lr_ratio": 0.041128435798161375, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 14, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.5121090187413508, + "rmse": 1.2296784208651264, + "mae": 0.5759273265230368, + "r2": 0.07455206011479953 + }, + "delivery": { + "n_samples": 58, + "mse": 0.5121668199501789, + "rmse": 0.7156583122902849, + "mae": 0.5583267274216331, + "r2": 0.34900506569110146 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7857142857142857, + "precision": 0.7385579937304074, + "recall": 0.7712045616535994, + "f1": 0.7493368700265252 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7380952380952381, + "precision": 0.6985851726089417, + "recall": 0.7376190476190475, + "f1": 0.7018460466736327 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9827586206896551, + "precision": 0.9912280701754386, + "recall": 0.75, + "f1": 0.8289085545722714 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.23962038921814616, + "js_divergence": 0.043880608373387 + } + } + }, + { + "fold": 1, + "best_params": { + "dropout": 0.23085562232445592, + "lr": 0.0005227270589225132, + "weight_decay": 0.00461701039547356, + "backbone_lr_ratio": 0.12087164274488635, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 7, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.3284213709926875, + "rmse": 0.5730805972921151, + "mae": 0.45920564744266723, + "r2": -0.002882105767083143 + }, + "delivery": { + "n_samples": 61, + "mse": 1.0560295226310608, + "rmse": 1.0276329707785075, + "mae": 0.6466741458070083, + "r2": 0.2006617794084804 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6666666666666666, + "precision": 0.5886939571150097, + "recall": 0.6031746031746031, + "f1": 0.5916666666666667 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5833333333333334, + "precision": 0.5271672771672772, + "recall": 0.533918462489891, + "f1": 0.513073133762789 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.4648032547075186, + "js_divergence": 0.11675330313943091 + } + } + }, + { + "fold": 2, + "best_params": { + "dropout": 0.11129874930397193, + "lr": 0.000993106647576575, + "weight_decay": 0.0015528212338926973, + "backbone_lr_ratio": 0.19356048624251188, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 10, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.395384611648953, + "rmse": 0.6287961606506142, + "mae": 0.4870017299295536, + "r2": 0.219122456317145 + }, + "delivery": { + "n_samples": 60, + "mse": 0.692732678583736, + "rmse": 0.8323056401249099, + "mae": 0.5703436779246355, + "r2": 0.07785273767480438 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7023809523809523, + "precision": 0.6503347534996957, + "recall": 0.6810850439882699, + "f1": 0.6554552912223134 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6547619047619048, + "precision": 0.6078622482131254, + "recall": 0.6720354808590102, + "f1": 0.6212949538882847 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.179209086609629, + "js_divergence": 0.0456761395398062 + } + } + }, + { + "fold": 3, + "best_params": { + "dropout": 0.19585998114210643, + "lr": 0.0008949481903172137, + "weight_decay": 0.0848478556523433, + "backbone_lr_ratio": 0.7944603821606866, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 11, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.7351497225620842, + "rmse": 1.317250819913233, + "mae": 0.577601661647569, + "r2": 0.018586091772742508 + }, + "delivery": { + "n_samples": 59, + "mse": 0.6791751469481081, + "rmse": 0.8241208327351688, + "mae": 0.5204403357959148, + "r2": 0.3397610637519505 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7142857142857143, + "precision": 0.6375, + "recall": 0.6451612903225806, + "f1": 0.6407697790449038 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6309523809523809, + "precision": 0.5632266806539034, + "recall": 0.5736299648064355, + "f1": 0.5568911622620062 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9166666666666666, + "precision": 0.6875, + "recall": 0.956140350877193, + "f1": 0.749791492910759 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.17262215641467948, + "js_divergence": 0.04320988831477907 + } + } + }, + { + "fold": 4, + "best_params": { + "dropout": 0.22523077425267193, + "lr": 0.0007489877504872848, + "weight_decay": 1.4840953126290499e-05, + "backbone_lr_ratio": 0.07616131024702885, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 13, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.5768887598177137, + "rmse": 0.7595319346924879, + "mae": 0.4391206152282566, + "r2": 0.22805118060144536 + }, + "delivery": { + "n_samples": 58, + "mse": 0.923829878979012, + "rmse": 0.9611606936298488, + "mae": 0.6288489371152788, + "r2": 0.1066717577051215 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6904761904761905, + "precision": 0.6333333333333333, + "recall": 0.6583577712609971, + "f1": 0.6379310344827587 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6904761904761905, + "precision": 0.6374458874458875, + "recall": 0.676068376068376, + "f1": 0.6504784466740988 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.8659090909090909 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.26043390803767386, + "js_divergence": 0.07162115663497268 + } + } + } + ], + "summary_stats": { + "size": { + "mse_mean": 0.9095906967525579, + "mse_std": 0.5928629344848133, + "rmse_mean": 0.9016675866827153, + "rmse_std": 0.3107833005139207, + "mae_mean": 0.5077713961542166, + "mae_std": 0.058351413912694466, + "r2_mean": 0.10748593660780985, + "r2_std": 0.09815051502898975 + }, + "delivery": { + "mse_mean": 0.7727868094184192, + "mse_std": 0.1929430120916135, + "rmse_mean": 0.8721756899117441, + "rmse_std": 0.10998352306319617, + "mae_mean": 0.5849267648128941, + "mae_std": 0.04652015524299455, + "r2_mean": 0.21479048084629165, + "r2_std": 0.11337762158357426 + }, + "pdi": { + "accuracy_mean": 0.7119047619047618, + "accuracy_std": 0.040124522729411236, + "precision_mean": 0.6496840075356892, + "precision_std": 0.04906035003802354, + "recall_mean": 0.67179665408001, + "recall_std": 0.05579827653185203, + "f1_mean": 0.6550319282886334, + "f1_std": 0.05177761509144909 + }, + "ee": { + "accuracy_mean": 0.6595238095238095, + "accuracy_std": 0.052489065916782394, + "precision_mean": 0.606857453217827, + "precision_std": 0.0593376860008789, + "recall_mean": 0.638654266368552, + "recall_std": 0.07416993652578822, + "f1_mean": 0.6087167486521623, + "f1_std": 0.0669461116008982 + }, + "toxic": { + "accuracy_mean": 0.9468758882575938, + "accuracy_std": 0.023999113331584828, + "precision_mean": 0.7814598997493734, + "precision_std": 0.11153441243627586, + "recall_mean": 0.9238635381557341, + "recall_std": 0.08733483101897535, + "f1_mean": 0.8017789705355671, + "f1_std": 0.04079837839243099 + }, + "biodist": { + "kl_divergence_mean": 0.2633377589975294, + "kl_divergence_std": 0.10626970030167675, + "js_divergence_mean": 0.06422821920047517, + "js_divergence_std": 0.028332014139597596 + } + } +} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_0/best_params.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_0/best_params.json new file mode 100644 index 0000000..c8e152d --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_0/best_params.json @@ -0,0 +1,12 @@ +{ + "dropout": 0.249816047538945, + "lr": 0.0007969454818643932, + "weight_decay": 0.008471801418819975, + "backbone_lr_ratio": 0.15751320499779725, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_0/epoch_mean.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_0/epoch_mean.json new file mode 100644 index 0000000..0ac5b47 --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_0/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 13} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_0/history.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_0/history.json new file mode 100644 index 0000000..69b92a4 --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_0/history.json @@ -0,0 +1,279 @@ +{ + "train": [ + { + "loss": 5.357723209593031, + "loss_size": 0.8442333026064767, + "loss_pdi": 0.6877147853374481, + "loss_ee": 1.0614066786236234, + "loss_delivery": 0.9942255467176437, + "loss_biodist": 1.178305768304401, + "loss_toxic": 0.601014655497339 + }, + { + "loss": 4.578087634510464, + "loss_size": 0.7912883642647002, + "loss_pdi": 0.645857592423757, + "loss_ee": 0.9956687125894759, + "loss_delivery": 0.9210033383634355, + "loss_biodist": 0.810003270705541, + "loss_toxic": 0.4352475206057231 + }, + { + "loss": 4.05100843641493, + "loss_size": 0.7856845764650239, + "loss_pdi": 0.6234016219774882, + "loss_ee": 0.9244379864798652, + "loss_delivery": 0.8481564356221093, + "loss_biodist": 0.5657288448678123, + "loss_toxic": 0.3257669061422348 + }, + { + "loss": 3.8016681008868747, + "loss_size": 0.7784977050291167, + "loss_pdi": 0.5801020049386554, + "loss_ee": 0.8832851615216997, + "loss_delivery": 0.9852454778220918, + "loss_biodist": 0.41591192450788284, + "loss_toxic": 0.18737825565040112 + }, + { + "loss": 3.340129190021091, + "loss_size": 0.7535917941066954, + "loss_pdi": 0.541335908903016, + "loss_ee": 0.8345312741067674, + "loss_delivery": 0.8228695392608643, + "loss_biodist": 0.3190598984559377, + "loss_toxic": 0.09481021482497454 + }, + { + "loss": 3.2055821418762207, + "loss_size": 0.6669195310937034, + "loss_pdi": 0.517667219042778, + "loss_ee": 0.781872977813085, + "loss_delivery": 0.8226976071794828, + "loss_biodist": 0.2737441228495704, + "loss_toxic": 0.16697360295802355 + }, + { + "loss": 3.091656188170115, + "loss_size": 0.6269183465176158, + "loss_pdi": 0.5062670012315115, + "loss_ee": 0.7809762954711914, + "loss_delivery": 0.8187296821011437, + "loss_biodist": 0.2924929228093889, + "loss_toxic": 0.09630843231247531 + }, + { + "loss": 2.8216485447353787, + "loss_size": 0.540225630833043, + "loss_pdi": 0.4871342678864797, + "loss_ee": 0.7632685601711273, + "loss_delivery": 0.7704160436987877, + "loss_biodist": 0.2361980966395802, + "loss_toxic": 0.05477537250974112 + }, + { + "loss": 2.80227890279558, + "loss_size": 0.5322931098441283, + "loss_pdi": 0.49128231075074935, + "loss_ee": 0.7562970866759618, + "loss_delivery": 0.7490749715103043, + "loss_biodist": 0.22268134810858303, + "loss_toxic": 0.08182886491219203 + }, + { + "loss": 2.7469793094529047, + "loss_size": 0.5142505344831281, + "loss_pdi": 0.47567099498377907, + "loss_ee": 0.7272394133938683, + "loss_delivery": 0.7498886121643914, + "loss_biodist": 0.2153525489072005, + "loss_toxic": 0.09466040061993732 + }, + { + "loss": 2.6655548016230264, + "loss_size": 0.5255163436134657, + "loss_pdi": 0.4635360886653264, + "loss_ee": 0.7227312955591414, + "loss_delivery": 0.7167085177368588, + "loss_biodist": 0.2128489592836963, + "loss_toxic": 0.056057278687755265 + }, + { + "loss": 2.6831478344069586, + "loss_size": 0.5698929462167952, + "loss_pdi": 0.45808757179313236, + "loss_ee": 0.7288233952389823, + "loss_delivery": 0.7129737378822433, + "loss_biodist": 0.18608077077402008, + "loss_toxic": 0.05943529913201928 + }, + { + "loss": 2.65675644742118, + "loss_size": 0.4960780491431554, + "loss_pdi": 0.47111381424797905, + "loss_ee": 0.7544109175602595, + "loss_delivery": 0.710324009673463, + "loss_biodist": 0.20193345348040262, + "loss_toxic": 0.05723577352344162 + } + ], + "val": [ + { + "loss": 4.618253389994304, + "loss_size": 0.4318694168080886, + "loss_pdi": 0.7372643351554871, + "loss_ee": 0.9695671598116556, + "loss_delivery": 0.6897582511107127, + "loss_biodist": 0.9869714180628458, + "loss_toxic": 0.8233492573102316, + "acc_pdi": 0.42424242424242425, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.75 + }, + { + "loss": 4.134536822636922, + "loss_size": 0.4229384424785773, + "loss_pdi": 0.8222103714942932, + "loss_ee": 0.8931615551312765, + "loss_delivery": 0.6330625961224238, + "loss_biodist": 0.6622798840204874, + "loss_toxic": 0.7266798317432404, + "acc_pdi": 0.3939393939393939, + "acc_ee": 0.48484848484848486, + "acc_toxic": 0.75 + }, + { + "loss": 3.9727744261423745, + "loss_size": 0.3946424846847852, + "loss_pdi": 0.9311431050300598, + "loss_ee": 0.8216386834780375, + "loss_delivery": 0.5963112314542135, + "loss_biodist": 0.5261518955230713, + "loss_toxic": 0.7307682037353516, + "acc_pdi": 0.45454545454545453, + "acc_ee": 0.5151515151515151, + "acc_toxic": 0.9 + }, + { + "loss": 3.5989553928375244, + "loss_size": 0.30280446928615373, + "loss_pdi": 1.121782620747884, + "loss_ee": 0.6880843428273996, + "loss_delivery": 0.47917167594035465, + "loss_biodist": 0.3424389412005742, + "loss_toxic": 0.6885180721680323, + "acc_pdi": 0.45454545454545453, + "acc_ee": 0.5151515151515151, + "acc_toxic": 0.9 + }, + { + "loss": 4.000288089116414, + "loss_size": 0.3686537444591522, + "loss_pdi": 1.28599880139033, + "loss_ee": 0.6944690197706223, + "loss_delivery": 0.5942268967628479, + "loss_biodist": 0.28115211178859073, + "loss_toxic": 0.8046019040048122, + "acc_pdi": 0.5757575757575758, + "acc_ee": 0.45454545454545453, + "acc_toxic": 0.9 + }, + { + "loss": 3.3209773699442544, + "loss_size": 0.3674306372801463, + "loss_pdi": 1.3185733358065288, + "loss_ee": 0.7143146321177483, + "loss_delivery": 0.4987892881035805, + "loss_biodist": 0.24983186274766922, + "loss_toxic": 0.21774169305960336, + "acc_pdi": 0.6363636363636364, + "acc_ee": 0.48484848484848486, + "acc_toxic": 1.0 + }, + { + "loss": 3.678060452143351, + "loss_size": 0.4425049424171448, + "loss_pdi": 1.387962559858958, + "loss_ee": 0.7107709869742393, + "loss_delivery": 0.46063798169294995, + "loss_biodist": 0.26061445847153664, + "loss_toxic": 0.45666170741120976, + "acc_pdi": 0.5151515151515151, + "acc_ee": 0.48484848484848486, + "acc_toxic": 0.9 + }, + { + "loss": 3.7740230560302734, + "loss_size": 0.45113707582155865, + "loss_pdi": 1.3036309878031414, + "loss_ee": 0.7190169282257557, + "loss_delivery": 0.533971885840098, + "loss_biodist": 0.23226053267717361, + "loss_toxic": 0.5684790350496769, + "acc_pdi": 0.5757575757575758, + "acc_ee": 0.48484848484848486, + "acc_toxic": 0.9 + }, + { + "loss": 3.318864901860555, + "loss_size": 0.410610834757487, + "loss_pdi": 1.211582859357198, + "loss_ee": 0.687928669154644, + "loss_delivery": 0.47218644122282666, + "loss_biodist": 0.20004537453254065, + "loss_toxic": 0.37275070138275623, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.48484848484848486, + "acc_toxic": 0.9 + }, + { + "loss": 3.3538697560628257, + "loss_size": 0.41582222779591876, + "loss_pdi": 1.2958858013153076, + "loss_ee": 0.703834168612957, + "loss_delivery": 0.4719666813810666, + "loss_biodist": 0.1989238758881887, + "loss_toxic": 0.3115424445519845, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.48484848484848486, + "acc_toxic": 0.9 + }, + { + "loss": 3.396747350692749, + "loss_size": 0.43390148878097534, + "loss_pdi": 1.3683980107307434, + "loss_ee": 0.6975282039493322, + "loss_delivery": 0.48154962062835693, + "loss_biodist": 0.20945363864302635, + "loss_toxic": 0.2563215708360076, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.48484848484848486, + "acc_toxic": 1.0 + }, + { + "loss": 3.367940902709961, + "loss_size": 0.4250214795271556, + "loss_pdi": 1.3399736483891804, + "loss_ee": 0.6810729236652454, + "loss_delivery": 0.4596002201239268, + "loss_biodist": 0.2265523001551628, + "loss_toxic": 0.28257635949800414, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.48484848484848486, + "acc_toxic": 0.9 + }, + { + "loss": 3.4425734678904214, + "loss_size": 0.43353842695554096, + "loss_pdi": 1.3702373902002971, + "loss_ee": 0.6958446235706409, + "loss_delivery": 0.4958720753590266, + "loss_biodist": 0.21138796831170717, + "loss_toxic": 0.2851239539061983, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.48484848484848486, + "acc_toxic": 0.9 + } + ] +} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_0/splits.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_0/splits.json new file mode 100644 index 0000000..cccb9d8 --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_0/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 52, 53, 54, 55, 56, 57, 58, 61, 62, 63, 64, 65, 66, 67, 68, 71, 72, 73, 75, 76, 77, 80, 81, 85, 86, 88, 89, 90, 92, 94, 95, 96, 98, 99, 100, 101, 103, 104, 105, 106, 107, 108, 109, 111, 113, 114, 115, 116, 117, 118, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 133, 134, 135, 136, 138, 139, 141, 144, 145, 146, 147, 148, 149, 150, 151, 153, 154, 155, 156, 157, 158, 159, 160, 163, 164, 167, 168, 169, 170, 171, 172, 175, 176, 178, 179, 180, 182, 183, 184, 185, 188, 189, 190, 191, 192, 193, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 206, 208, 210, 211, 212, 213, 216, 217, 218, 219, 220, 222, 224, 225, 227, 228, 229, 230, 231, 232, 233, 234, 236, 237, 239, 241, 242, 244, 245, 247, 248, 249, 250, 252, 253, 255, 256, 257, 258, 259, 260, 261, 262, 265, 267, 268, 269, 270, 271, 272, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 285, 286, 287, 288, 291, 292, 294, 296, 297, 298, 299, 300, 301, 303, 304, 306, 307, 308, 309, 310, 311, 312, 313, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 332, 333, 335, 336, 337, 338, 339, 340, 342, 343, 344, 345, 346, 347, 348, 349, 351, 352, 354, 355, 356, 357, 358, 359, 360, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 374, 375, 377, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 393, 395, 396, 397, 398, 399, 400, 401, 403, 404, 405, 406, 407, 408, 409, 411, 412, 413, 414, 415, 417, 418, 419], "outer_test_idx": [7, 16, 20, 29, 51, 59, 60, 69, 70, 74, 78, 79, 82, 83, 84, 87, 91, 93, 97, 102, 110, 112, 119, 120, 121, 132, 137, 140, 142, 143, 152, 161, 162, 165, 166, 173, 174, 177, 181, 186, 187, 194, 205, 207, 209, 214, 215, 221, 223, 226, 235, 238, 240, 243, 246, 251, 254, 263, 264, 266, 273, 284, 289, 290, 293, 295, 302, 305, 314, 315, 331, 334, 341, 350, 353, 361, 373, 376, 378, 392, 394, 402, 410, 416]} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_0/test_metrics.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_0/test_metrics.json new file mode 100644 index 0000000..4bcde74 --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_0/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 83, + "mse": 1.4997791464662995, + "rmse": 1.2246547049949628, + "mae": 0.5329577082610992, + "r2": 0.08209824544573019 + }, + "delivery": { + "n_samples": 58, + "mse": 0.5621633803139553, + "rmse": 0.7497755532917536, + "mae": 0.5971168833463614, + "r2": 0.2854564204804384 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7976190476190477, + "precision": 0.7458333333333333, + "recall": 0.7523164647184604, + "f1": 0.7489010022859153 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.75, + "precision": 0.728583244712277, + "recall": 0.7614285714285715, + "f1": 0.7299803816657748 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9655172413793104, + "precision": 0.75, + "recall": 0.9821428571428572, + "f1": 0.8242424242424242 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.2505519774721038, + "js_divergence": 0.04876761665965193 + } +} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_1/best_params.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_1/best_params.json new file mode 100644 index 0000000..8dad003 --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_1/best_params.json @@ -0,0 +1,12 @@ +{ + "dropout": 0.36125999893855604, + "lr": 0.0005218589746733537, + "weight_decay": 0.00013957295237475344, + "backbone_lr_ratio": 0.05368136611253809, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_1/epoch_mean.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_1/epoch_mean.json new file mode 100644 index 0000000..54e2f4a --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_1/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 15} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_1/history.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_1/history.json new file mode 100644 index 0000000..06e3b63 --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_1/history.json @@ -0,0 +1,321 @@ +{ + "train": [ + { + "loss": 5.911894003550212, + "loss_size": 1.250612893866168, + "loss_pdi": 0.6854646768834856, + "loss_ee": 1.0637510650687747, + "loss_delivery": 0.9928857717249129, + "loss_biodist": 1.2614385618103876, + "loss_toxic": 0.665780868795183 + }, + { + "loss": 5.437452144092983, + "loss_size": 1.214371142288049, + "loss_pdi": 0.6780849132272933, + "loss_ee": 1.0095830725298987, + "loss_delivery": 0.9280333444476128, + "loss_biodist": 1.0316998925473955, + "loss_toxic": 0.5973956485589346 + }, + { + "loss": 4.99566031826867, + "loss_size": 1.134289861553245, + "loss_pdi": 0.6639255351490445, + "loss_ee": 0.9786890480253432, + "loss_delivery": 0.8746029966407352, + "loss_biodist": 0.8240409228536818, + "loss_toxic": 0.5482145051161448 + }, + { + "loss": 4.217750708262126, + "loss_size": 0.7819488677713606, + "loss_pdi": 0.6369739638434516, + "loss_ee": 0.9398350285159217, + "loss_delivery": 0.8110355337460836, + "loss_biodist": 0.61703436407778, + "loss_toxic": 0.4552278270324071 + }, + { + "loss": 4.367909550666809, + "loss_size": 1.1233808965318732, + "loss_pdi": 0.6312973068820106, + "loss_ee": 0.901738123761283, + "loss_delivery": 0.87216607398457, + "loss_biodist": 0.49617693821589154, + "loss_toxic": 0.36838628434472614 + }, + { + "loss": 4.053940441873339, + "loss_size": 1.096034876174397, + "loss_pdi": 0.6121236996518241, + "loss_ee": 0.8876421981387668, + "loss_delivery": 0.7170077785849571, + "loss_biodist": 0.4290698750151528, + "loss_toxic": 0.33317113088236916 + }, + { + "loss": 3.8955683840645685, + "loss_size": 1.1155037639869585, + "loss_pdi": 0.581842937403255, + "loss_ee": 0.8709939585791694, + "loss_delivery": 0.7423711203866534, + "loss_biodist": 0.3659255231420199, + "loss_toxic": 0.2386209625336859 + }, + { + "loss": 3.8433918820487127, + "loss_size": 1.047416937847932, + "loss_pdi": 0.5826017409563065, + "loss_ee": 0.8429841697216034, + "loss_delivery": 0.8056356824106641, + "loss_biodist": 0.35747069120407104, + "loss_toxic": 0.22848275262448522 + }, + { + "loss": 3.8038561079237194, + "loss_size": 1.0617801224191983, + "loss_pdi": 0.5594437172015508, + "loss_ee": 0.8418945968151093, + "loss_delivery": 0.7961080140537686, + "loss_biodist": 0.303602514995469, + "loss_toxic": 0.26238997528950375 + }, + { + "loss": 3.7466025352478027, + "loss_size": 1.0623111724853516, + "loss_pdi": 0.5658913105726242, + "loss_ee": 0.8407420847151015, + "loss_delivery": 0.7677783792217573, + "loss_biodist": 0.3127972384293874, + "loss_toxic": 0.21951221550504366 + }, + { + "loss": 3.6121728155348034, + "loss_size": 1.0435051197806995, + "loss_pdi": 0.562022070089976, + "loss_ee": 0.7908639477358924, + "loss_delivery": 0.7580708207355605, + "loss_biodist": 0.27753299267755616, + "loss_toxic": 0.20060701130164993 + }, + { + "loss": 3.5683264334996543, + "loss_size": 1.060364168551233, + "loss_pdi": 0.5438821597231759, + "loss_ee": 0.8113437791665395, + "loss_delivery": 0.7331801578402519, + "loss_biodist": 0.2752979364660051, + "loss_toxic": 0.16549518559541967 + }, + { + "loss": 3.539075016975403, + "loss_size": 1.0168713455398877, + "loss_pdi": 0.5293869210614098, + "loss_ee": 0.8257469716999266, + "loss_delivery": 0.681364561120669, + "loss_biodist": 0.26487314618296093, + "loss_toxic": 0.24054492140809694 + }, + { + "loss": 3.5629933873812356, + "loss_size": 0.946415576669905, + "loss_pdi": 0.5580547600984573, + "loss_ee": 0.8289351264635721, + "loss_delivery": 0.8261245596739981, + "loss_biodist": 0.25930900623401004, + "loss_toxic": 0.16812019629610908 + }, + { + "loss": 3.6414078341590033, + "loss_size": 1.0167089675863583, + "loss_pdi": 0.5789187202850977, + "loss_ee": 0.8134810792075263, + "loss_delivery": 0.7805551638205847, + "loss_biodist": 0.2700353190302849, + "loss_toxic": 0.20578757528629568 + } + ], + "val": [ + { + "loss": 3.9704039891560874, + "loss_size": 0.49022617885687697, + "loss_pdi": 0.6732653776804606, + "loss_ee": 1.0454180637995403, + "loss_delivery": 0.24724662800629935, + "loss_biodist": 0.9668523669242859, + "loss_toxic": 0.5571489334106445, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.696969696969697, + "acc_toxic": 1.0 + }, + { + "loss": 3.431459665298462, + "loss_size": 0.47595814714441076, + "loss_pdi": 0.6395421624183655, + "loss_ee": 0.9424608945846558, + "loss_delivery": 0.2004812154918909, + "loss_biodist": 0.7341917554537455, + "loss_toxic": 0.4482741554578145, + "acc_pdi": 0.6060606060606061, + "acc_ee": 0.696969696969697, + "acc_toxic": 1.0 + }, + { + "loss": 3.045083920160929, + "loss_size": 0.44765157128373784, + "loss_pdi": 0.6216420531272888, + "loss_ee": 0.8668653964996338, + "loss_delivery": 0.16731029618070656, + "loss_biodist": 0.5658134917418162, + "loss_toxic": 0.3810176948706309, + "acc_pdi": 0.6363636363636364, + "acc_ee": 0.6666666666666666, + "acc_toxic": 1.0 + }, + { + "loss": 2.6452345848083496, + "loss_size": 0.5306090911229452, + "loss_pdi": 0.5736951132615408, + "loss_ee": 0.7957356572151184, + "loss_delivery": 0.18814919392267862, + "loss_biodist": 0.3778227965037028, + "loss_toxic": 0.17461387316385904, + "acc_pdi": 0.6060606060606061, + "acc_ee": 0.6666666666666666, + "acc_toxic": 1.0 + }, + { + "loss": 2.4595271746317544, + "loss_size": 0.5288840234279633, + "loss_pdi": 0.5541006326675415, + "loss_ee": 0.7416675885518392, + "loss_delivery": 0.23529979586601257, + "loss_biodist": 0.3138190855582555, + "loss_toxic": 0.07786177222927411, + "acc_pdi": 0.6060606060606061, + "acc_ee": 0.6666666666666666, + "acc_toxic": 1.0 + }, + { + "loss": 2.465927561124166, + "loss_size": 0.5028193593025208, + "loss_pdi": 0.5479741791884104, + "loss_ee": 0.701051394144694, + "loss_delivery": 0.3289036105076472, + "loss_biodist": 0.3326733609040578, + "loss_toxic": 0.046381489063302674, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.7272727272727273, + "acc_toxic": 1.0 + }, + { + "loss": 2.503588159879049, + "loss_size": 0.5209598938624064, + "loss_pdi": 0.5699935356775919, + "loss_ee": 0.706265409787496, + "loss_delivery": 0.36016706625620526, + "loss_biodist": 0.3159407575925191, + "loss_toxic": 0.02749400306493044, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.7272727272727273, + "acc_toxic": 1.0 + }, + { + "loss": 2.5452259381612143, + "loss_size": 0.5663575232028961, + "loss_pdi": 0.5590367317199707, + "loss_ee": 0.6887768010298411, + "loss_delivery": 0.3837193151315053, + "loss_biodist": 0.3254129116733869, + "loss_toxic": 0.0214148194839557, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.7575757575757576, + "acc_toxic": 1.0 + }, + { + "loss": 2.5326576232910156, + "loss_size": 0.5633496443430582, + "loss_pdi": 0.5706869463125864, + "loss_ee": 0.6523322562376658, + "loss_delivery": 0.3952372074127197, + "loss_biodist": 0.33602823813756305, + "loss_toxic": 0.015840193411956232, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.696969696969697, + "acc_toxic": 1.0 + }, + { + "loss": 2.5047752062479653, + "loss_size": 0.5343820850054423, + "loss_pdi": 0.5656164288520813, + "loss_ee": 0.6412633806467056, + "loss_delivery": 0.4159560402234395, + "loss_biodist": 0.3340985005100568, + "loss_toxic": 0.015062282715613643, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.7575757575757576, + "acc_toxic": 1.0 + }, + { + "loss": 2.510882536570231, + "loss_size": 0.5417856077353159, + "loss_pdi": 0.5540410975615183, + "loss_ee": 0.6391958643992742, + "loss_delivery": 0.4346950153509776, + "loss_biodist": 0.3305298909544945, + "loss_toxic": 0.013345858780667186, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.7575757575757576, + "acc_toxic": 1.0 + }, + { + "loss": 2.46993350982666, + "loss_size": 0.524556557337443, + "loss_pdi": 0.5535958707332611, + "loss_ee": 0.6416819194952647, + "loss_delivery": 0.4096498688062032, + "loss_biodist": 0.3299781357248624, + "loss_toxic": 0.01324144572330018, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.7272727272727273, + "acc_toxic": 1.0 + }, + { + "loss": 2.418788274129232, + "loss_size": 0.5074512958526611, + "loss_pdi": 0.5515439311663309, + "loss_ee": 0.61900062362353, + "loss_delivery": 0.39639513691266376, + "loss_biodist": 0.3329038793841998, + "loss_toxic": 0.013266399269923568, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.7272727272727273, + "acc_toxic": 1.0 + }, + { + "loss": 2.4144439697265625, + "loss_size": 0.5023249288400015, + "loss_pdi": 0.5596796870231628, + "loss_ee": 0.6254272510608038, + "loss_delivery": 0.38848791519800824, + "loss_biodist": 0.32847215235233307, + "loss_toxic": 0.012236134226744374, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.696969696969697, + "acc_toxic": 1.0 + }, + { + "loss": 2.445017417271932, + "loss_size": 0.5287560820579529, + "loss_pdi": 0.5598017871379852, + "loss_ee": 0.6349637309710184, + "loss_delivery": 0.37778013944625854, + "loss_biodist": 0.33295177668333054, + "loss_toxic": 0.013552555426334342, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.7575757575757576, + "acc_toxic": 1.0 + } + ] +} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_1/splits.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_1/splits.json new file mode 100644 index 0000000..1773fe5 --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_1/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [0, 3, 5, 6, 7, 10, 11, 12, 13, 15, 16, 18, 19, 20, 21, 24, 26, 27, 29, 30, 31, 32, 33, 35, 36, 38, 39, 40, 42, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 65, 66, 67, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 97, 98, 99, 100, 101, 102, 103, 105, 106, 107, 108, 109, 110, 112, 114, 115, 118, 119, 120, 121, 123, 124, 125, 126, 128, 129, 130, 131, 132, 136, 137, 140, 141, 142, 143, 144, 145, 146, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 161, 162, 164, 165, 166, 167, 168, 169, 171, 172, 173, 174, 175, 176, 177, 178, 179, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 194, 195, 196, 197, 198, 199, 201, 202, 204, 205, 206, 207, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 225, 226, 228, 229, 231, 232, 233, 234, 235, 236, 237, 238, 240, 242, 243, 244, 246, 247, 248, 250, 251, 252, 254, 255, 256, 258, 259, 260, 261, 262, 263, 264, 266, 268, 269, 270, 271, 273, 274, 275, 276, 277, 278, 279, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 295, 296, 297, 300, 301, 302, 303, 304, 305, 306, 307, 308, 310, 311, 313, 314, 315, 316, 317, 318, 319, 321, 322, 323, 324, 325, 327, 328, 329, 330, 331, 333, 334, 335, 337, 338, 339, 340, 341, 342, 343, 344, 345, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 361, 362, 364, 365, 366, 367, 368, 369, 373, 374, 375, 376, 378, 379, 380, 381, 382, 384, 387, 388, 389, 391, 392, 393, 394, 396, 397, 398, 399, 402, 403, 404, 405, 406, 408, 409, 410, 411, 412, 413, 414, 416, 417, 419], "outer_test_idx": [1, 2, 4, 8, 9, 14, 17, 22, 23, 25, 28, 34, 37, 41, 43, 45, 63, 64, 68, 85, 96, 104, 111, 113, 116, 117, 122, 127, 133, 134, 135, 138, 139, 147, 160, 163, 170, 180, 191, 192, 193, 200, 203, 208, 224, 227, 230, 239, 241, 245, 249, 253, 257, 265, 267, 272, 280, 294, 298, 299, 309, 312, 320, 326, 332, 336, 346, 347, 360, 363, 370, 371, 372, 377, 383, 385, 386, 390, 395, 400, 401, 407, 415, 418]} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_1/test_metrics.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_1/test_metrics.json new file mode 100644 index 0000000..82a3995 --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_1/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.3450189290389537, + "rmse": 0.5873831194705494, + "mae": 0.45183507455069394, + "r2": -0.053565147232133015 + }, + "delivery": { + "n_samples": 61, + "mse": 1.1082149368704723, + "rmse": 1.0527178809493416, + "mae": 0.6711371511404143, + "r2": 0.1611611828200118 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6547619047619048, + "precision": 0.5696286472148542, + "recall": 0.5793650793650793, + "f1": 0.571654650958326 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5595238095238095, + "precision": 0.50171998624011, + "recall": 0.5036154321868608, + "f1": 0.4913136288998358 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9508196721311475, + "precision": 0.75, + "recall": 0.9741379310344828, + "f1": 0.8200589970501475 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.26572251656260437, + "js_divergence": 0.06909315114786709 + } +} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_2/best_params.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_2/best_params.json new file mode 100644 index 0000000..122743e --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_2/best_params.json @@ -0,0 +1,12 @@ +{ + "dropout": 0.497804687444555, + "lr": 0.0008262030137574559, + "weight_decay": 0.0027980162766859887, + "backbone_lr_ratio": 0.25041980149691795, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_2/epoch_mean.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_2/epoch_mean.json new file mode 100644 index 0000000..735a6b9 --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_2/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 11} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_2/history.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_2/history.json new file mode 100644 index 0000000..59aa749 --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_2/history.json @@ -0,0 +1,237 @@ +{ + "train": [ + { + "loss": 5.512474377950032, + "loss_size": 0.8958140992456012, + "loss_pdi": 0.6994819210635291, + "loss_ee": 1.0986503627565172, + "loss_delivery": 0.9981613043281767, + "loss_biodist": 1.1998903022872076, + "loss_toxic": 0.6345649361610413 + }, + { + "loss": 4.946902884377374, + "loss_size": 0.857138411866294, + "loss_pdi": 0.6753449903594123, + "loss_ee": 1.0291849209202661, + "loss_delivery": 0.9593191138572164, + "loss_biodist": 0.8999736640188429, + "loss_toxic": 0.5574632121456994 + }, + { + "loss": 4.502587146229214, + "loss_size": 0.8356625586748123, + "loss_pdi": 0.6691454682085249, + "loss_ee": 0.9701411061816745, + "loss_delivery": 0.9051497164699767, + "loss_biodist": 0.6777517961131202, + "loss_toxic": 0.48112630512979293 + }, + { + "loss": 4.111242188347711, + "loss_size": 0.8308768098553022, + "loss_pdi": 0.6555037498474121, + "loss_ee": 0.974882099363539, + "loss_delivery": 0.783248268895679, + "loss_biodist": 0.4677625795205434, + "loss_toxic": 0.43308165421088535 + }, + { + "loss": 3.508241335550944, + "loss_size": 0.8114278722140524, + "loss_pdi": 0.5958007524410883, + "loss_ee": 0.9036731157037947, + "loss_delivery": 0.5716096692615085, + "loss_biodist": 0.3424467369914055, + "loss_toxic": 0.302171290334728 + }, + { + "loss": 3.395925177468194, + "loss_size": 0.7918954152199957, + "loss_pdi": 0.5876384758287005, + "loss_ee": 0.8857002357641856, + "loss_delivery": 0.6633324954244826, + "loss_biodist": 0.2992335061232249, + "loss_toxic": 0.1880970417211453 + }, + { + "loss": 3.4075717793570623, + "loss_size": 0.7641554102301598, + "loss_pdi": 0.5451137406958474, + "loss_ee": 0.849973632229699, + "loss_delivery": 0.832838834159904, + "loss_biodist": 0.29037230130698943, + "loss_toxic": 0.14845767824186218 + }, + { + "loss": 3.312739862336053, + "loss_size": 0.7040707377923859, + "loss_pdi": 0.5588378459215164, + "loss_ee": 0.8407626118924882, + "loss_delivery": 0.8211448523733351, + "loss_biodist": 0.27182113958729637, + "loss_toxic": 0.14081234619435337 + }, + { + "loss": 3.1818973620732627, + "loss_size": 0.6639444472061263, + "loss_pdi": 0.5625713401370578, + "loss_ee": 0.8262495050827662, + "loss_delivery": 0.7170704205830892, + "loss_biodist": 0.26208172490199405, + "loss_toxic": 0.17187502866403925 + }, + { + "loss": 3.2750312752193875, + "loss_size": 0.6916999369859695, + "loss_pdi": 0.5635132806168662, + "loss_ee": 0.8544793128967285, + "loss_delivery": 0.8074588949481646, + "loss_biodist": 0.2660403781467014, + "loss_toxic": 0.11973961566885312 + }, + { + "loss": 3.115817348162333, + "loss_size": 0.6864229010211097, + "loss_pdi": 0.5672376337978575, + "loss_ee": 0.7979768349064721, + "loss_delivery": 0.6953879677587085, + "loss_biodist": 0.2522631469700072, + "loss_toxic": 0.13790533484684098 + } + ], + "val": [ + { + "loss": 7.1353128751118975, + "loss_size": 3.0783068537712097, + "loss_pdi": 0.7280304829279581, + "loss_ee": 1.0537604888280232, + "loss_delivery": 1.3098933696746826, + "loss_biodist": 0.7146384914716085, + "loss_toxic": 0.2889313797156016, + "acc_pdi": 0.45454545454545453, + "acc_ee": 0.45454545454545453, + "acc_toxic": 1.0 + }, + { + "loss": 7.156964461008708, + "loss_size": 3.420055458943049, + "loss_pdi": 0.7195907433827718, + "loss_ee": 1.0802890261014302, + "loss_delivery": 1.313367744286855, + "loss_biodist": 0.4719923933347066, + "loss_toxic": 0.21157964070638022, + "acc_pdi": 0.45454545454545453, + "acc_ee": 0.5454545454545454, + "acc_toxic": 1.0 + }, + { + "loss": 7.20744530359904, + "loss_size": 3.5315067768096924, + "loss_pdi": 0.7128446698188782, + "loss_ee": 1.1136252681414287, + "loss_delivery": 1.397851785024007, + "loss_biodist": 0.3658549189567566, + "loss_toxic": 0.16914697488149008, + "acc_pdi": 0.48484848484848486, + "acc_ee": 0.5151515151515151, + "acc_toxic": 0.95 + }, + { + "loss": 7.512194871902466, + "loss_size": 3.7619529167811074, + "loss_pdi": 0.687969962755839, + "loss_ee": 1.3171117107073467, + "loss_delivery": 1.560650457938512, + "loss_biodist": 0.16003490487734476, + "loss_toxic": 0.12764343122641245, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.95 + }, + { + "loss": 7.195699294408162, + "loss_size": 3.5671945412953696, + "loss_pdi": 0.6796847581863403, + "loss_ee": 1.195267120997111, + "loss_delivery": 1.5593087474505107, + "loss_biodist": 0.15528676410516104, + "loss_toxic": 0.14114744464556375, + "acc_pdi": 0.48484848484848486, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.95 + }, + { + "loss": 6.9594011306762695, + "loss_size": 3.507280786832174, + "loss_pdi": 0.6869745055834452, + "loss_ee": 1.1664539376894634, + "loss_delivery": 1.4385378162066143, + "loss_biodist": 0.13912539184093475, + "loss_toxic": 0.12253157173593839, + "acc_pdi": 0.6363636363636364, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.95 + }, + { + "loss": 7.096203565597534, + "loss_size": 3.563037017981211, + "loss_pdi": 0.6828234195709229, + "loss_ee": 1.2166488369305928, + "loss_delivery": 1.4491861859957378, + "loss_biodist": 0.14899996419747671, + "loss_toxic": 0.1448717142144839, + "acc_pdi": 0.6363636363636364, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.95 + }, + { + "loss": 6.818722724914551, + "loss_size": 3.460448980331421, + "loss_pdi": 0.6628589232762655, + "loss_ee": 1.1797481775283813, + "loss_delivery": 1.3693017562230427, + "loss_biodist": 0.12642602622509003, + "loss_toxic": 0.1251833445082108, + "acc_pdi": 0.5454545454545454, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.95 + }, + { + "loss": 6.703790664672852, + "loss_size": 3.4174140294392905, + "loss_pdi": 0.6559697389602661, + "loss_ee": 1.1504637598991394, + "loss_delivery": 1.3375890652338664, + "loss_biodist": 0.13071069618066153, + "loss_toxic": 0.11497477690378825, + "acc_pdi": 0.5454545454545454, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.95 + }, + { + "loss": 6.598912000656128, + "loss_size": 3.321854372819265, + "loss_pdi": 0.6576323707898458, + "loss_ee": 1.1500487128893535, + "loss_delivery": 1.3335067232449849, + "loss_biodist": 0.12197343011697133, + "loss_toxic": 0.11658582153419654, + "acc_pdi": 0.5151515151515151, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.95 + }, + { + "loss": 6.560119072596232, + "loss_size": 3.3144603967666626, + "loss_pdi": 0.6507784326871237, + "loss_ee": 1.1165754795074463, + "loss_delivery": 1.3492673635482788, + "loss_biodist": 0.11476222177346547, + "loss_toxic": 0.11514309110740821, + "acc_pdi": 0.5454545454545454, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.95 + } + ] +} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_2/splits.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_2/splits.json new file mode 100644 index 0000000..14df306 --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_2/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 36, 37, 38, 39, 40, 41, 43, 44, 45, 47, 48, 49, 50, 51, 55, 56, 57, 58, 59, 60, 61, 63, 64, 65, 68, 69, 70, 72, 74, 76, 78, 79, 80, 82, 83, 84, 85, 86, 87, 88, 91, 93, 94, 95, 96, 97, 100, 101, 102, 103, 104, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 142, 143, 145, 147, 148, 149, 150, 151, 152, 153, 156, 157, 158, 159, 160, 161, 162, 163, 165, 166, 167, 168, 170, 171, 172, 173, 174, 175, 177, 179, 180, 181, 183, 184, 185, 186, 187, 188, 191, 192, 193, 194, 196, 197, 199, 200, 202, 203, 204, 205, 207, 208, 209, 210, 211, 212, 213, 214, 215, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 232, 233, 235, 236, 237, 238, 239, 240, 241, 242, 243, 245, 246, 249, 250, 251, 252, 253, 254, 255, 256, 257, 259, 260, 261, 263, 264, 265, 266, 267, 268, 269, 271, 272, 273, 274, 275, 276, 280, 281, 282, 283, 284, 285, 287, 289, 290, 293, 294, 295, 296, 297, 298, 299, 300, 302, 303, 305, 307, 308, 309, 310, 311, 312, 313, 314, 315, 317, 318, 320, 321, 322, 323, 324, 326, 327, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 358, 360, 361, 363, 364, 365, 366, 368, 369, 370, 371, 372, 373, 376, 377, 378, 379, 380, 381, 382, 383, 385, 386, 387, 388, 389, 390, 392, 394, 395, 398, 399, 400, 401, 402, 404, 405, 407, 408, 409, 410, 412, 413, 414, 415, 416, 417, 418], "outer_test_idx": [10, 12, 15, 31, 35, 42, 46, 52, 53, 54, 62, 66, 67, 71, 73, 75, 77, 81, 89, 90, 92, 98, 99, 105, 123, 124, 141, 144, 146, 154, 155, 164, 169, 176, 178, 182, 189, 190, 195, 198, 201, 206, 216, 217, 218, 219, 231, 234, 244, 247, 248, 258, 262, 270, 277, 278, 279, 286, 288, 291, 292, 301, 304, 306, 316, 319, 325, 328, 343, 357, 359, 362, 367, 374, 375, 384, 391, 393, 396, 397, 403, 406, 411, 419]} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_2/test_metrics.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_2/test_metrics.json new file mode 100644 index 0000000..fe0578b --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_2/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.33659527812850365, + "rmse": 0.5801683187907658, + "mae": 0.4592306216779564, + "r2": 0.3352303396329478 + }, + "delivery": { + "n_samples": 60, + "mse": 0.6215280245003019, + "rmse": 0.7883704868272923, + "mae": 0.5425195595948026, + "r2": 0.17263847372827446 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.5952380952380952, + "precision": 0.6068181818181818, + "recall": 0.6378299120234604, + "f1": 0.5757575757575757 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6547619047619048, + "precision": 0.606140350877193, + "recall": 0.6720354808590102, + "f1": 0.6190881743565225 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.2123447981522219, + "js_divergence": 0.04886770818690606 + } +} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_3/best_params.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_3/best_params.json new file mode 100644 index 0000000..d1be4ee --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_3/best_params.json @@ -0,0 +1,12 @@ +{ + "dropout": 0.19620664816030634, + "lr": 0.0008203303040781945, + "weight_decay": 0.09163011132541928, + "backbone_lr_ratio": 0.7546459184154254, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_3/epoch_mean.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_3/epoch_mean.json new file mode 100644 index 0000000..84179ef --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_3/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 12} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_3/history.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_3/history.json new file mode 100644 index 0000000..16af224 --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_3/history.json @@ -0,0 +1,258 @@ +{ + "train": [ + { + "loss": 4.917902814017402, + "loss_size": 0.4309130261341731, + "loss_pdi": 0.6868788335058424, + "loss_ee": 1.0647370318571727, + "loss_delivery": 0.9559381674561236, + "loss_biodist": 1.1634221739239163, + "loss_toxic": 0.6274078918827904 + }, + { + "loss": 4.166169444719951, + "loss_size": 0.4625764961043994, + "loss_pdi": 0.6466403537326388, + "loss_ee": 0.9935492840078142, + "loss_delivery": 0.867659674750434, + "loss_biodist": 0.7590122520923615, + "loss_toxic": 0.45675261484252083 + }, + { + "loss": 3.5520550674862332, + "loss_size": 0.42359134430686635, + "loss_pdi": 0.6201435029506683, + "loss_ee": 0.9346062607235379, + "loss_delivery": 0.7960347640845511, + "loss_biodist": 0.47750501500235665, + "loss_toxic": 0.32001306861639023 + }, + { + "loss": 3.5057511064741345, + "loss_size": 0.4692589019735654, + "loss_pdi": 0.605718731880188, + "loss_ee": 0.9070098698139191, + "loss_delivery": 0.8530328886376487, + "loss_biodist": 0.37685007684760624, + "loss_toxic": 0.31892701362570125 + }, + { + "loss": 3.0965431994862027, + "loss_size": 0.46113158679670757, + "loss_pdi": 0.5877639883094363, + "loss_ee": 0.8644410338666704, + "loss_delivery": 0.7830151617527008, + "loss_biodist": 0.3081934087806278, + "loss_toxic": 0.11731299913177888 + }, + { + "loss": 2.833913962046305, + "loss_size": 0.41022086722983253, + "loss_pdi": 0.5255969381994672, + "loss_ee": 0.8318477438555824, + "loss_delivery": 0.7059815807474984, + "loss_biodist": 0.2562926552361912, + "loss_toxic": 0.1280389095708314 + }, + { + "loss": 2.6549873683187695, + "loss_size": 0.4042331708802117, + "loss_pdi": 0.5605269405576918, + "loss_ee": 0.8348930958244536, + "loss_delivery": 0.5936605119042926, + "loss_biodist": 0.20830887431899706, + "loss_toxic": 0.081134278818758 + }, + { + "loss": 2.489447944694095, + "loss_size": 0.35215044187174904, + "loss_pdi": 0.4786165522204505, + "loss_ee": 0.738041157523791, + "loss_delivery": 0.6567219396432241, + "loss_biodist": 0.20787620544433594, + "loss_toxic": 0.0822850492162009 + }, + { + "loss": 2.437808487150404, + "loss_size": 0.3821040752033393, + "loss_pdi": 0.4929063204261992, + "loss_ee": 0.6996884213553535, + "loss_delivery": 0.6443286149038209, + "loss_biodist": 0.16231619566679, + "loss_toxic": 0.08252853763082789 + }, + { + "loss": 2.2655367255210876, + "loss_size": 0.3315175088743369, + "loss_pdi": 0.4602713104751375, + "loss_ee": 0.6495821608437432, + "loss_delivery": 0.6121829839216338, + "loss_biodist": 0.15002488013770846, + "loss_toxic": 0.08624541882373807 + }, + { + "loss": 2.1731932825512357, + "loss_size": 0.350161112844944, + "loss_pdi": 0.4589882526132796, + "loss_ee": 0.6437088118659126, + "loss_delivery": 0.53968455725246, + "loss_biodist": 0.1482078093621466, + "loss_toxic": 0.056304057348623045 + }, + { + "loss": 2.0813635455237494, + "loss_size": 0.31070471327337956, + "loss_pdi": 0.4515724976857503, + "loss_ee": 0.6381474120749367, + "loss_delivery": 0.5166368488636282, + "loss_biodist": 0.1431189796162976, + "loss_toxic": 0.04592720178576807 + } + ], + "val": [ + { + "loss": 5.768293698628743, + "loss_size": 2.281007265051206, + "loss_pdi": 0.701847493648529, + "loss_ee": 0.9739474852879842, + "loss_delivery": 0.8268086115519205, + "loss_biodist": 0.660688598950704, + "loss_toxic": 0.3274596134821574, + "acc_pdi": 0.45454545454545453, + "acc_ee": 0.5757575757575758, + "acc_toxic": 1.0 + }, + { + "loss": 5.159181674321492, + "loss_size": 2.3360865910847983, + "loss_pdi": 0.627079168955485, + "loss_ee": 0.8099661469459534, + "loss_delivery": 0.7186701248089472, + "loss_biodist": 0.35371384024620056, + "loss_toxic": 0.3010466893513997, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.7575757575757576, + "acc_toxic": 0.782608695652174 + }, + { + "loss": 4.74508261680603, + "loss_size": 2.293394833803177, + "loss_pdi": 0.6011759638786316, + "loss_ee": 0.7338899572690328, + "loss_delivery": 0.64650097489357, + "loss_biodist": 0.20491118729114532, + "loss_toxic": 0.24377821882565817, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.6666666666666666, + "acc_toxic": 0.782608695652174 + }, + { + "loss": 4.492343902587891, + "loss_size": 2.136273900667826, + "loss_pdi": 0.5764371554056803, + "loss_ee": 0.7934263249238332, + "loss_delivery": 0.6320122083028158, + "loss_biodist": 0.228810985883077, + "loss_toxic": 0.10945691168308258, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 4.379851261774699, + "loss_size": 1.8871766552329063, + "loss_pdi": 0.5225809613863627, + "loss_ee": 0.906223197778066, + "loss_delivery": 0.7094851732254028, + "loss_biodist": 0.11288597683111827, + "loss_toxic": 0.23303943872451782, + "acc_pdi": 0.8181818181818182, + "acc_ee": 0.696969696969697, + "acc_toxic": 0.8260869565217391 + }, + { + "loss": 3.9695494771003723, + "loss_size": 1.9443345566590626, + "loss_pdi": 0.4245029091835022, + "loss_ee": 0.7277062137921652, + "loss_delivery": 0.5624868273735046, + "loss_biodist": 0.1416376680135727, + "loss_toxic": 0.1455500324567159, + "acc_pdi": 0.8181818181818182, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 4.15172811349233, + "loss_size": 1.9145114123821259, + "loss_pdi": 0.5264913241068522, + "loss_ee": 0.7966079811255137, + "loss_delivery": 0.6179003169139227, + "loss_biodist": 0.11703654130299886, + "loss_toxic": 0.16525722543398538, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.6363636363636364, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 4.125605722268422, + "loss_size": 1.967104842265447, + "loss_pdi": 0.5304079651832581, + "loss_ee": 0.777589331070582, + "loss_delivery": 0.5806996325651804, + "loss_biodist": 0.1155442198117574, + "loss_toxic": 0.1377676526705424, + "acc_pdi": 0.7878787878787878, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 3.9747113784154258, + "loss_size": 1.8420614997545879, + "loss_pdi": 0.49178969860076904, + "loss_ee": 0.7782185971736908, + "loss_delivery": 0.6073268353939056, + "loss_biodist": 0.10780937969684601, + "loss_toxic": 0.13335004448890686, + "acc_pdi": 0.8181818181818182, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 3.721254587173462, + "loss_size": 1.669212852915128, + "loss_pdi": 0.4937492112318675, + "loss_ee": 0.7702797055244446, + "loss_delivery": 0.6004098455111185, + "loss_biodist": 0.11377307772636414, + "loss_toxic": 0.06692169855038325, + "acc_pdi": 0.7878787878787878, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.9565217391304348 + }, + { + "loss": 3.7344598372777305, + "loss_size": 1.7442348798116047, + "loss_pdi": 0.4792054096857707, + "loss_ee": 0.7417815923690796, + "loss_delivery": 0.5857466955979665, + "loss_biodist": 0.09634749591350555, + "loss_toxic": 0.07474951694409053, + "acc_pdi": 0.7878787878787878, + "acc_ee": 0.6666666666666666, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 3.6552441517512, + "loss_size": 1.6857635130484898, + "loss_pdi": 0.48044103384017944, + "loss_ee": 0.7329937219619751, + "loss_delivery": 0.5694670875867208, + "loss_biodist": 0.09875263770421346, + "loss_toxic": 0.07628875474135081, + "acc_pdi": 0.7878787878787878, + "acc_ee": 0.6363636363636364, + "acc_toxic": 0.8695652173913043 + } + ] +} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_3/splits.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_3/splits.json new file mode 100644 index 0000000..7a2b13e --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_3/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [0, 1, 2, 4, 5, 7, 8, 9, 10, 11, 12, 14, 15, 16, 17, 20, 21, 22, 23, 24, 25, 27, 28, 29, 30, 31, 32, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 45, 46, 47, 51, 52, 53, 54, 55, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 73, 74, 75, 76, 77, 78, 79, 81, 82, 83, 84, 85, 86, 87, 89, 90, 91, 92, 93, 95, 96, 97, 98, 99, 100, 102, 104, 105, 106, 109, 110, 111, 112, 113, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 127, 129, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 146, 147, 148, 150, 151, 152, 153, 154, 155, 156, 158, 159, 160, 161, 162, 163, 164, 165, 166, 168, 169, 170, 172, 173, 174, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 189, 190, 191, 192, 193, 194, 195, 198, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 212, 213, 214, 215, 216, 217, 218, 219, 221, 223, 224, 226, 227, 228, 230, 231, 233, 234, 235, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 251, 252, 253, 254, 255, 256, 257, 258, 259, 262, 263, 264, 265, 266, 267, 268, 269, 270, 272, 273, 274, 275, 276, 277, 278, 279, 280, 282, 284, 286, 288, 289, 290, 291, 292, 293, 294, 295, 297, 298, 299, 301, 302, 304, 305, 306, 307, 308, 309, 310, 312, 314, 315, 316, 317, 319, 320, 321, 325, 326, 327, 328, 331, 332, 334, 336, 340, 341, 343, 344, 346, 347, 348, 350, 352, 353, 357, 359, 360, 361, 362, 363, 364, 366, 367, 368, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 382, 383, 384, 385, 386, 387, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 400, 401, 402, 403, 404, 406, 407, 410, 411, 412, 414, 415, 416, 418, 419], "outer_test_idx": [3, 6, 13, 18, 19, 26, 33, 44, 48, 49, 50, 56, 57, 72, 80, 88, 94, 101, 103, 107, 108, 114, 126, 128, 130, 145, 149, 157, 167, 171, 175, 188, 196, 197, 199, 211, 220, 222, 225, 229, 232, 236, 250, 260, 261, 271, 281, 283, 285, 287, 296, 300, 303, 311, 313, 318, 322, 323, 324, 329, 330, 333, 335, 337, 338, 339, 342, 345, 349, 351, 354, 355, 356, 358, 365, 369, 381, 388, 399, 405, 408, 409, 413, 417]} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_3/test_metrics.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_3/test_metrics.json new file mode 100644 index 0000000..22b559c --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_3/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 83, + "mse": 1.7611879007938644, + "rmse": 1.3270975475803821, + "mae": 0.5631435849782394, + "r2": 0.0038586997043298377 + }, + "delivery": { + "n_samples": 59, + "mse": 0.6930136672963602, + "rmse": 0.8324744244097594, + "mae": 0.541700589438995, + "r2": 0.32630837780631017 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6072124756335282, + "recall": 0.6209677419354839, + "f1": 0.6110444177671068 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6428571428571429, + "precision": 0.5801358234295416, + "recall": 0.599270990447461, + "f1": 0.5749640680242134 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9333333333333333, + "precision": 0.7142857142857143, + "recall": 0.9649122807017544, + "f1": 0.7818181818181817 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.20239779905020466, + "js_divergence": 0.050275046923259875 + } +} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_4/best_params.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_4/best_params.json new file mode 100644 index 0000000..2ea1e2f --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_4/best_params.json @@ -0,0 +1,12 @@ +{ + "dropout": 0.3061951110950902, + "lr": 0.0008643881414012139, + "weight_decay": 0.00011771973478525925, + "backbone_lr_ratio": 0.12472198351350613, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_4/epoch_mean.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_4/epoch_mean.json new file mode 100644 index 0000000..54e2f4a --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_4/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 15} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_4/history.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_4/history.json new file mode 100644 index 0000000..5bac6e7 --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_4/history.json @@ -0,0 +1,321 @@ +{ + "train": [ + { + "loss": 5.2031600210401745, + "loss_size": 0.8160930027564367, + "loss_pdi": 0.6898496448993683, + "loss_ee": 1.0792100495762296, + "loss_delivery": 0.8426996403270297, + "loss_biodist": 1.147640390528573, + "loss_toxic": 0.6392683519257439 + }, + { + "loss": 4.856290645069546, + "loss_size": 1.1223950940701697, + "loss_pdi": 0.663935919602712, + "loss_ee": 1.0117611289024353, + "loss_delivery": 0.8356892375482453, + "loss_biodist": 0.7835476249456406, + "loss_toxic": 0.4617169267601437 + }, + { + "loss": 4.198579178916083, + "loss_size": 1.0817829722331629, + "loss_pdi": 0.6312899854448106, + "loss_ee": 0.9564476841025882, + "loss_delivery": 0.6959641401966413, + "loss_biodist": 0.502743569513162, + "loss_toxic": 0.35363391786813736 + }, + { + "loss": 3.839665479130215, + "loss_size": 1.048507057958179, + "loss_pdi": 0.5924790253241857, + "loss_ee": 0.929336451821857, + "loss_delivery": 0.702328828887807, + "loss_biodist": 0.37197959588633644, + "loss_toxic": 0.22153377895139986 + }, + { + "loss": 3.541697528627184, + "loss_size": 1.0175944252146616, + "loss_pdi": 0.5758830789062712, + "loss_ee": 0.8467480639616648, + "loss_delivery": 0.7099834589494599, + "loss_biodist": 0.2953973164161046, + "loss_toxic": 0.12227233209543759 + }, + { + "loss": 3.3752357562383017, + "loss_size": 0.9258440981308619, + "loss_pdi": 0.5514006714026133, + "loss_ee": 0.8352020747131772, + "loss_delivery": 0.7375599717100462, + "loss_biodist": 0.23662048702438673, + "loss_toxic": 0.1176186182225744 + }, + { + "loss": 3.0509555339813232, + "loss_size": 0.8627384793427255, + "loss_pdi": 0.5291877074374093, + "loss_ee": 0.8081569208039178, + "loss_delivery": 0.5561069233549966, + "loss_biodist": 0.2129895691242483, + "loss_toxic": 0.10808462287402815 + }, + { + "loss": 3.093853341208564, + "loss_size": 0.865282324453195, + "loss_pdi": 0.5258895175324546, + "loss_ee": 0.7829923811886046, + "loss_delivery": 0.6627265256312158, + "loss_biodist": 0.19228361174464226, + "loss_toxic": 0.09675240154481596 + }, + { + "loss": 2.995564619700114, + "loss_size": 0.8384040863149695, + "loss_pdi": 0.5372706005970637, + "loss_ee": 0.7907316121790144, + "loss_delivery": 0.6236173626449373, + "loss_biodist": 0.19440977896253267, + "loss_toxic": 0.04907751642167568 + }, + { + "loss": 2.9788292249043784, + "loss_size": 0.7636799597077899, + "loss_pdi": 0.5124389330546061, + "loss_ee": 0.8097357584370507, + "loss_delivery": 0.6510066617694166, + "loss_biodist": 0.19554469941390884, + "loss_toxic": 0.08602368275427984 + }, + { + "loss": 2.910417550139957, + "loss_size": 0.8150636839369932, + "loss_pdi": 0.5073161572217941, + "loss_ee": 0.7562909358077579, + "loss_delivery": 0.6424184263580375, + "loss_biodist": 0.17400132657753098, + "loss_toxic": 0.05588890899283191 + }, + { + "loss": 2.693137672212389, + "loss_size": 0.7773789415756861, + "loss_pdi": 0.502211794257164, + "loss_ee": 0.7380809254116483, + "loss_delivery": 0.48622770938608384, + "loss_biodist": 0.17270910491545996, + "loss_toxic": 0.0532212672341201 + }, + { + "loss": 2.8423878153165183, + "loss_size": 0.7224535559200578, + "loss_pdi": 0.4856733762555652, + "loss_ee": 0.7525817983680301, + "loss_delivery": 0.6508662543363042, + "loss_biodist": 0.17564408315552604, + "loss_toxic": 0.09449486918230024 + }, + { + "loss": 2.8017438848813376, + "loss_size": 0.7341417753034167, + "loss_pdi": 0.4987424562374751, + "loss_ee": 0.744825025399526, + "loss_delivery": 0.6140849052204026, + "loss_biodist": 0.18327445619636112, + "loss_toxic": 0.06761009939428833 + }, + { + "loss": 2.8013548188739352, + "loss_size": 0.6857024903098742, + "loss_pdi": 0.4601053380303913, + "loss_ee": 0.7546558048990037, + "loss_delivery": 0.5786655222376188, + "loss_biodist": 0.1816521059307787, + "loss_toxic": 0.1726232705016931 + } + ], + "val": [ + { + "loss": 4.718395233154297, + "loss_size": 0.68744428952535, + "loss_pdi": 0.5974234342575073, + "loss_ee": 1.0439125299453735, + "loss_delivery": 0.6930600467603654, + "loss_biodist": 1.0893035928408306, + "loss_toxic": 0.628725012143453, + "acc_pdi": 0.3939393939393939, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 3.7455027103424072, + "loss_size": 0.4566015402475993, + "loss_pdi": 0.5505313177903494, + "loss_ee": 0.837164560953776, + "loss_delivery": 0.6726585874954859, + "loss_biodist": 0.6968304812908173, + "loss_toxic": 0.5506061812241873, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 3.2456515630086265, + "loss_size": 0.38795870542526245, + "loss_pdi": 0.5055133799711863, + "loss_ee": 0.7072862486044565, + "loss_delivery": 0.6790636777877808, + "loss_biodist": 0.4258381724357605, + "loss_toxic": 0.5490268617868423, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 2.94810160001119, + "loss_size": 0.24454845984776816, + "loss_pdi": 0.39873725175857544, + "loss_ee": 0.648169718682766, + "loss_delivery": 0.8554844955603281, + "loss_biodist": 0.19254801670710245, + "loss_toxic": 0.607252286747098, + "acc_pdi": 0.5757575757575758, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 3.157598892847697, + "loss_size": 0.24779518942038217, + "loss_pdi": 0.37889902542034787, + "loss_ee": 0.6869482571880022, + "loss_delivery": 0.755667415757974, + "loss_biodist": 0.15918351461489996, + "loss_toxic": 0.92043586820364, + "acc_pdi": 0.5757575757575758, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 2.888409376144409, + "loss_size": 0.3273507257302602, + "loss_pdi": 0.37816034505764645, + "loss_ee": 0.6894322273631891, + "loss_delivery": 0.8181691418091456, + "loss_biodist": 0.13849087059497833, + "loss_toxic": 0.5366462791959444, + "acc_pdi": 0.6060606060606061, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 3.0905685424804688, + "loss_size": 0.3234970470269521, + "loss_pdi": 0.3679231715699037, + "loss_ee": 0.6802832366277775, + "loss_delivery": 0.7895760791337428, + "loss_biodist": 0.12661263098319372, + "loss_toxic": 0.7926427914450566, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 2.5253223975499473, + "loss_size": 0.2613599896430969, + "loss_pdi": 0.36543068538109463, + "loss_ee": 0.6754304027805725, + "loss_delivery": 0.7046491494402289, + "loss_biodist": 0.11572940150896709, + "loss_toxic": 0.40416052502890426, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.5262065728505454, + "loss_size": 0.23982082307338715, + "loss_pdi": 0.36069128662347794, + "loss_ee": 0.7058537335445484, + "loss_delivery": 0.7162311447318643, + "loss_biodist": 0.09738401075204213, + "loss_toxic": 0.4101589284837246, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.6124425133069358, + "loss_size": 0.2753596305847168, + "loss_pdi": 0.36195003738005954, + "loss_ee": 0.7248491427550713, + "loss_delivery": 0.7471474803363284, + "loss_biodist": 0.10086645185947418, + "loss_toxic": 0.41014073168238, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.635942498842875, + "loss_size": 0.290947546561559, + "loss_pdi": 0.3645387254655361, + "loss_ee": 0.7260850742459297, + "loss_delivery": 0.7620968722427884, + "loss_biodist": 0.09960449983676274, + "loss_toxic": 0.4027190475414197, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.711034377415975, + "loss_size": 0.34289781252543133, + "loss_pdi": 0.35783156131704646, + "loss_ee": 0.7489802089209358, + "loss_delivery": 0.7772864030363659, + "loss_biodist": 0.09336689362923305, + "loss_toxic": 0.4033478343238433, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.7135236263275146, + "loss_size": 0.34678372740745544, + "loss_pdi": 0.36285089204708737, + "loss_ee": 0.7369768234590689, + "loss_delivery": 0.7855381645883123, + "loss_biodist": 0.09028732279936473, + "loss_toxic": 0.4039084513982137, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.6510766744613647, + "loss_size": 0.30508745710055035, + "loss_pdi": 0.36125267545382184, + "loss_ee": 0.7261834535747766, + "loss_delivery": 0.7729004731712242, + "loss_biodist": 0.09505814562241237, + "loss_toxic": 0.4022646602243185, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.661883592605591, + "loss_size": 0.31253430247306824, + "loss_pdi": 0.3626324087381363, + "loss_ee": 0.7328057729949554, + "loss_delivery": 0.7720068310211445, + "loss_biodist": 0.09150024006764094, + "loss_toxic": 0.40260524302721024, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.9444444444444444 + } + ] +} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_4/splits.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_4/splits.json new file mode 100644 index 0000000..8dd0299 --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_4/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23, 25, 26, 28, 29, 31, 33, 34, 35, 37, 41, 42, 43, 44, 45, 46, 48, 49, 50, 51, 52, 53, 54, 56, 57, 59, 60, 62, 63, 64, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 77, 78, 79, 80, 81, 82, 83, 84, 85, 87, 88, 89, 90, 91, 92, 93, 94, 96, 97, 98, 99, 101, 102, 103, 104, 105, 107, 108, 110, 111, 112, 113, 114, 116, 117, 119, 120, 121, 122, 123, 124, 126, 127, 128, 130, 132, 133, 134, 135, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 149, 152, 154, 155, 157, 160, 161, 162, 163, 164, 165, 166, 167, 169, 170, 171, 173, 174, 175, 176, 177, 178, 180, 181, 182, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 203, 205, 206, 207, 208, 209, 211, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 229, 230, 231, 232, 234, 235, 236, 238, 239, 240, 241, 243, 244, 245, 246, 247, 248, 249, 250, 251, 253, 254, 257, 258, 260, 261, 262, 263, 264, 265, 266, 267, 270, 271, 272, 273, 277, 278, 279, 280, 281, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 298, 299, 300, 301, 302, 303, 304, 305, 306, 309, 311, 312, 313, 314, 315, 316, 318, 319, 320, 322, 323, 324, 325, 326, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 341, 342, 343, 345, 346, 347, 349, 350, 351, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 365, 367, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 381, 383, 384, 385, 386, 388, 390, 391, 392, 393, 394, 395, 396, 397, 399, 400, 401, 402, 403, 405, 406, 407, 408, 409, 410, 411, 413, 415, 416, 417, 418, 419], "outer_test_idx": [0, 5, 11, 21, 24, 27, 30, 32, 36, 38, 39, 40, 47, 55, 58, 61, 65, 76, 86, 95, 100, 106, 109, 115, 118, 125, 129, 131, 136, 148, 150, 151, 153, 156, 158, 159, 168, 172, 179, 183, 184, 185, 202, 204, 210, 212, 213, 228, 233, 237, 242, 252, 255, 256, 259, 268, 269, 274, 275, 276, 282, 297, 307, 308, 310, 317, 321, 327, 340, 344, 348, 352, 364, 366, 368, 379, 380, 382, 387, 389, 398, 404, 412, 414]} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/outer_fold_4/test_metrics.json b/models/abl_full/s2_moleculestm/seed42/outer_fold_4/test_metrics.json new file mode 100644 index 0000000..cc80fac --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/outer_fold_4/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.5915837666928274, + "rmse": 0.769144828164909, + "mae": 0.4365386986783484, + "r2": 0.2083874360488871 + }, + "delivery": { + "n_samples": 58, + "mse": 0.9647128449414197, + "rmse": 0.9821979662682161, + "mae": 0.6298785273884905, + "r2": 0.06713860451964326 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7619047619047619, + "precision": 0.7175480769230769, + "recall": 0.7653958944281525, + "f1": 0.7270955165692008 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6904761904761905, + "precision": 0.6276353276353276, + "recall": 0.6547008547008547, + "f1": 0.6362408486478409 + }, + "toxic": { + "n_samples": 59, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.18118856500710467, + "js_divergence": 0.044863367372506804 + } +} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/strata_info.json b/models/abl_full/s2_moleculestm/seed42/strata_info.json new file mode 100644 index 0000000..ef3be58 --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/strata_info.json @@ -0,0 +1,60 @@ +{ + "original_strata_counts": { + "T0|P0|E0": "5", + "T0|P0|E1": "58", + "T0|P0|E2": "169", + "T0|P1|E0": "1", + "T0|P1|E1": "17", + "T0|P1|E2": "32", + "T0|P2|E2": "3", + "T1|P0|E2": "9", + "T1|P1|E2": "5", + "TNA|P0|E0": "28", + "TNA|P0|E1": "21", + "TNA|P0|E2": "20", + "TNA|P1|E0": "29", + "TNA|P1|E1": "7", + "TNA|P1|E2": "14", + "TNA|P2|E2": "1", + "TNA|P3|E0": "1" + }, + "rare_strata": [ + "T0|P1|E0", + "T0|P2|E2", + "TNA|P2|E2", + "TNA|P3|E0" + ], + "final_strata": [ + "RARE", + "T0|P0|E0", + "T0|P0|E1", + "T0|P0|E2", + "T0|P1|E1", + "T0|P1|E2", + "T1|P0|E2", + "T1|P1|E2", + "TNA|P0|E0", + "TNA|P0|E1", + "TNA|P0|E2", + "TNA|P1|E0", + "TNA|P1|E1", + "TNA|P1|E2" + ], + "final_strata_counts": { + "RARE": "6", + "T0|P0|E0": "5", + "T0|P0|E1": "58", + "T0|P0|E2": "169", + "T0|P1|E1": "17", + "T0|P1|E2": "32", + "T1|P0|E2": "9", + "T1|P1|E2": "5", + "TNA|P0|E0": "28", + "TNA|P0|E1": "21", + "TNA|P0|E2": "20", + "TNA|P1|E0": "29", + "TNA|P1|E1": "7", + "TNA|P1|E2": "14" + }, + "n_rare_merged": "6" +} \ No newline at end of file diff --git a/models/abl_full/s2_moleculestm/seed42/summary.json b/models/abl_full/s2_moleculestm/seed42/summary.json new file mode 100644 index 0000000..dd9f907 --- /dev/null +++ b/models/abl_full/s2_moleculestm/seed42/summary.json @@ -0,0 +1,352 @@ +{ + "fold_results": [ + { + "fold": 0, + "best_params": { + "dropout": 0.249816047538945, + "lr": 0.0007969454818643932, + "weight_decay": 0.008471801418819975, + "backbone_lr_ratio": 0.15751320499779725, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 13, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.4997791464662995, + "rmse": 1.2246547049949628, + "mae": 0.5329577082610992, + "r2": 0.08209824544573019 + }, + "delivery": { + "n_samples": 58, + "mse": 0.5621633803139553, + "rmse": 0.7497755532917536, + "mae": 0.5971168833463614, + "r2": 0.2854564204804384 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7976190476190477, + "precision": 0.7458333333333333, + "recall": 0.7523164647184604, + "f1": 0.7489010022859153 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.75, + "precision": 0.728583244712277, + "recall": 0.7614285714285715, + "f1": 0.7299803816657748 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9655172413793104, + "precision": 0.75, + "recall": 0.9821428571428572, + "f1": 0.8242424242424242 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.2505519774721038, + "js_divergence": 0.04876761665965193 + } + } + }, + { + "fold": 1, + "best_params": { + "dropout": 0.36125999893855604, + "lr": 0.0005218589746733537, + "weight_decay": 0.00013957295237475344, + "backbone_lr_ratio": 0.05368136611253809, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 15, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.3450189290389537, + "rmse": 0.5873831194705494, + "mae": 0.45183507455069394, + "r2": -0.053565147232133015 + }, + "delivery": { + "n_samples": 61, + "mse": 1.1082149368704723, + "rmse": 1.0527178809493416, + "mae": 0.6711371511404143, + "r2": 0.1611611828200118 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6547619047619048, + "precision": 0.5696286472148542, + "recall": 0.5793650793650793, + "f1": 0.571654650958326 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5595238095238095, + "precision": 0.50171998624011, + "recall": 0.5036154321868608, + "f1": 0.4913136288998358 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9508196721311475, + "precision": 0.75, + "recall": 0.9741379310344828, + "f1": 0.8200589970501475 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.26572251656260437, + "js_divergence": 0.06909315114786709 + } + } + }, + { + "fold": 2, + "best_params": { + "dropout": 0.497804687444555, + "lr": 0.0008262030137574559, + "weight_decay": 0.0027980162766859887, + "backbone_lr_ratio": 0.25041980149691795, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 11, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.33659527812850365, + "rmse": 0.5801683187907658, + "mae": 0.4592306216779564, + "r2": 0.3352303396329478 + }, + "delivery": { + "n_samples": 60, + "mse": 0.6215280245003019, + "rmse": 0.7883704868272923, + "mae": 0.5425195595948026, + "r2": 0.17263847372827446 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.5952380952380952, + "precision": 0.6068181818181818, + "recall": 0.6378299120234604, + "f1": 0.5757575757575757 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6547619047619048, + "precision": 0.606140350877193, + "recall": 0.6720354808590102, + "f1": 0.6190881743565225 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.2123447981522219, + "js_divergence": 0.04886770818690606 + } + } + }, + { + "fold": 3, + "best_params": { + "dropout": 0.19620664816030634, + "lr": 0.0008203303040781945, + "weight_decay": 0.09163011132541928, + "backbone_lr_ratio": 0.7546459184154254, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 12, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.7611879007938644, + "rmse": 1.3270975475803821, + "mae": 0.5631435849782394, + "r2": 0.0038586997043298377 + }, + "delivery": { + "n_samples": 59, + "mse": 0.6930136672963602, + "rmse": 0.8324744244097594, + "mae": 0.541700589438995, + "r2": 0.32630837780631017 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6072124756335282, + "recall": 0.6209677419354839, + "f1": 0.6110444177671068 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6428571428571429, + "precision": 0.5801358234295416, + "recall": 0.599270990447461, + "f1": 0.5749640680242134 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9333333333333333, + "precision": 0.7142857142857143, + "recall": 0.9649122807017544, + "f1": 0.7818181818181817 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.20239779905020466, + "js_divergence": 0.050275046923259875 + } + } + }, + { + "fold": 4, + "best_params": { + "dropout": 0.3061951110950902, + "lr": 0.0008643881414012139, + "weight_decay": 0.00011771973478525925, + "backbone_lr_ratio": 0.12472198351350613, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 15, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.5915837666928274, + "rmse": 0.769144828164909, + "mae": 0.4365386986783484, + "r2": 0.2083874360488871 + }, + "delivery": { + "n_samples": 58, + "mse": 0.9647128449414197, + "rmse": 0.9821979662682161, + "mae": 0.6298785273884905, + "r2": 0.06713860451964326 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7619047619047619, + "precision": 0.7175480769230769, + "recall": 0.7653958944281525, + "f1": 0.7270955165692008 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6904761904761905, + "precision": 0.6276353276353276, + "recall": 0.6547008547008547, + "f1": 0.6362408486478409 + }, + "toxic": { + "n_samples": 59, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.18118856500710467, + "js_divergence": 0.044863367372506804 + } + } + } + ], + "summary_stats": { + "size": { + "mse_mean": 0.9068330042240899, + "mse_std": 0.6036051056768003, + "rmse_mean": 0.8976897038003138, + "rmse_std": 0.3177832593372322, + "mae_mean": 0.4887411376292675, + "mae_std": 0.04989759907260662, + "r2_mean": 0.11520191471995238, + "r2_std": 0.14076330118636055 + }, + "delivery": { + "mse_mean": 0.7899265707845018, + "mse_std": 0.21046803928662908, + "rmse_mean": 0.8811072623492725, + "rmse_std": 0.11651850934453273, + "mae_mean": 0.5964705421818127, + "mae_std": 0.050203724904436454, + "r2_mean": 0.20254061187093564, + "r2_std": 0.09288012684190738 + }, + "pdi": { + "accuracy_mean": 0.6976190476190476, + "accuracy_std": 0.07323122142490333, + "precision_mean": 0.6494081429845948, + "precision_std": 0.06913756578237862, + "recall_mean": 0.6711750184941273, + "recall_std": 0.07419331428775373, + "f1_mean": 0.6468906326676249, + "f1_std": 0.07595295223116724 + }, + "ee": { + "accuracy_mean": 0.6595238095238095, + "accuracy_std": 0.062360956446232345, + "precision_mean": 0.6088429465788898, + "precision_std": 0.0734814591004088, + "recall_mean": 0.6382102659245517, + "recall_std": 0.08512446409627589, + "f1_mean": 0.6103174203188375, + "f1_std": 0.07807680381726986 + }, + "toxic": { + "accuracy_mean": 0.9568192952703976, + "accuracy_std": 0.024614072428637914, + "precision_mean": 0.7857142857142858, + "precision_std": 0.10832679205787929, + "recall_mean": 0.977342062051681, + "recall_std": 0.012969364591169584, + "f1_mean": 0.8416524920507221, + "f1_std": 0.08119712321141342 + }, + "biodist": { + "kl_divergence_mean": 0.22244113124884785, + "kl_divergence_std": 0.031205587061351694, + "js_divergence_mean": 0.05237337805803836, + "js_divergence_std": 0.008551553107252485 + } + } +} \ No newline at end of file diff --git a/models/abl_full/tanimoto_knn/seed42/outer_fold_0/test_metrics.json b/models/abl_full/tanimoto_knn/seed42/outer_fold_0/test_metrics.json new file mode 100644 index 0000000..cd84966 --- /dev/null +++ b/models/abl_full/tanimoto_knn/seed42/outer_fold_0/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 83, + "mse": 1.762732353807776, + "rmse": 1.3276793113578955, + "mae": 0.6979452131473156, + "r2": -0.07883558225954723 + }, + "delivery": { + "n_samples": 58, + "mse": 0.746449183391017, + "rmse": 0.8639729066301888, + "mae": 0.646006099558596, + "r2": 0.051218037285134965 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9827586206896551, + "precision": 0.9912280701754386, + "recall": 0.75, + "f1": 0.8289085545722714 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.5119047619047619, + "precision": 0.4791666666666667, + "recall": 0.47434069850320737, + "f1": 0.46446897838594314 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7261904761904762, + "precision": 0.6766726403823178, + "recall": 0.6523809523809524, + "f1": 0.641367232698502 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.34443429112434387, + "js_divergence": 0.08364559710025787 + } +} \ No newline at end of file diff --git a/models/abl_full/tanimoto_knn/seed42/outer_fold_1/test_metrics.json b/models/abl_full/tanimoto_knn/seed42/outer_fold_1/test_metrics.json new file mode 100644 index 0000000..71ea766 --- /dev/null +++ b/models/abl_full/tanimoto_knn/seed42/outer_fold_1/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.5181048119517085, + "rmse": 0.7197949791098216, + "mae": 0.5254934162256264, + "r2": -0.5821078937747561 + }, + "delivery": { + "n_samples": 61, + "mse": 1.1243467667528977, + "rmse": 1.0603521899599668, + "mae": 0.6867733584014607, + "r2": 0.14895057570408377 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9508196721311475, + "precision": 0.75, + "recall": 0.9741379310344828, + "f1": 0.8200589970501475 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7261904761904762, + "precision": 0.6071428571428571, + "recall": 0.5793650793650793, + "f1": 0.5849624060150376 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5952380952380952, + "precision": 0.5352564102564102, + "recall": 0.5345131059416773, + "f1": 0.5263532763532763 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.7807825803756714, + "js_divergence": 0.12687404453754425 + } +} \ No newline at end of file diff --git a/models/abl_full/tanimoto_knn/seed42/outer_fold_2/test_metrics.json b/models/abl_full/tanimoto_knn/seed42/outer_fold_2/test_metrics.json new file mode 100644 index 0000000..93046ce --- /dev/null +++ b/models/abl_full/tanimoto_knn/seed42/outer_fold_2/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.44417219953539205, + "rmse": 0.6664624517070651, + "mae": 0.5136315814618554, + "r2": 0.12276783858786378 + }, + "delivery": { + "n_samples": 60, + "mse": 0.6732209168800264, + "rmse": 0.8205004063862652, + "mae": 0.5596807344001717, + "r2": 0.10382628141775896 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9180327868852459, + "precision": 0.6875, + "recall": 0.9568965517241379, + "f1": 0.7502047502047502 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.75, + "precision": 0.6652173913043478, + "recall": 0.625366568914956, + "f1": 0.636063544460491 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7142857142857143, + "precision": 0.6787755102040816, + "recall": 0.6622315592903828, + "f1": 0.6671936758893281 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.3705731928348541, + "js_divergence": 0.0945676639676094 + } +} \ No newline at end of file diff --git a/models/abl_full/tanimoto_knn/seed42/outer_fold_3/test_metrics.json b/models/abl_full/tanimoto_knn/seed42/outer_fold_3/test_metrics.json new file mode 100644 index 0000000..ca8941b --- /dev/null +++ b/models/abl_full/tanimoto_knn/seed42/outer_fold_3/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 83, + "mse": 1.9476472096853905, + "rmse": 1.3955813160419535, + "mae": 0.6786922237201852, + "r2": -0.10160416549494755 + }, + "delivery": { + "n_samples": 59, + "mse": 1.0546120682688678, + "rmse": 1.0269430696337882, + "mae": 0.7278208352972643, + "r2": -0.025208277515615496 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9166666666666666, + "precision": 0.6875, + "recall": 0.956140350877193, + "f1": 0.749791492910759 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7738095238095238, + "precision": 0.7091007583965331, + "recall": 0.6414956011730205, + "f1": 0.6571428571428573 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6071428571428571, + "precision": 0.4344751866490997, + "recall": 0.443614881850176, + "f1": 0.4324842382393017 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.9395005106925964, + "js_divergence": 0.12415548413991928 + } +} \ No newline at end of file diff --git a/models/abl_full/tanimoto_knn/seed42/outer_fold_4/test_metrics.json b/models/abl_full/tanimoto_knn/seed42/outer_fold_4/test_metrics.json new file mode 100644 index 0000000..5e8c3da --- /dev/null +++ b/models/abl_full/tanimoto_knn/seed42/outer_fold_4/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.7430198511001034, + "rmse": 0.8619859924036488, + "mae": 0.5445426530481893, + "r2": 0.005747196126379084 + }, + "delivery": { + "n_samples": 58, + "mse": 1.0484304054163862, + "rmse": 1.0239289064268018, + "mae": 0.6575078364333202, + "r2": -0.013814952507488876 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.8659090909090909 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6666666666666666, + "precision": 0.5142857142857142, + "recall": 0.5102639296187683, + "f1": 0.5050505050505051 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6547619047619048, + "precision": 0.6187822362094589, + "recall": 0.6354700854700855, + "f1": 0.6152751423149906 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.4118382930755615, + "js_divergence": 0.1084776222705841 + } +} \ No newline at end of file diff --git a/models/abl_full/tanimoto_knn/seed42/summary.json b/models/abl_full/tanimoto_knn/seed42/summary.json new file mode 100644 index 0000000..d84d5c3 --- /dev/null +++ b/models/abl_full/tanimoto_knn/seed42/summary.json @@ -0,0 +1,287 @@ +{ + "fold_results": [ + { + "fold": 0, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.762732353807776, + "rmse": 1.3276793113578955, + "mae": 0.6979452131473156, + "r2": -0.07883558225954723 + }, + "delivery": { + "n_samples": 58, + "mse": 0.746449183391017, + "rmse": 0.8639729066301888, + "mae": 0.646006099558596, + "r2": 0.051218037285134965 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9827586206896551, + "precision": 0.9912280701754386, + "recall": 0.75, + "f1": 0.8289085545722714 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.5119047619047619, + "precision": 0.4791666666666667, + "recall": 0.47434069850320737, + "f1": 0.46446897838594314 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7261904761904762, + "precision": 0.6766726403823178, + "recall": 0.6523809523809524, + "f1": 0.641367232698502 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.34443429112434387, + "js_divergence": 0.08364559710025787 + } + } + }, + { + "fold": 1, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.5181048119517085, + "rmse": 0.7197949791098216, + "mae": 0.5254934162256264, + "r2": -0.5821078937747561 + }, + "delivery": { + "n_samples": 61, + "mse": 1.1243467667528977, + "rmse": 1.0603521899599668, + "mae": 0.6867733584014607, + "r2": 0.14895057570408377 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9508196721311475, + "precision": 0.75, + "recall": 0.9741379310344828, + "f1": 0.8200589970501475 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7261904761904762, + "precision": 0.6071428571428571, + "recall": 0.5793650793650793, + "f1": 0.5849624060150376 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5952380952380952, + "precision": 0.5352564102564102, + "recall": 0.5345131059416773, + "f1": 0.5263532763532763 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.7807825803756714, + "js_divergence": 0.12687404453754425 + } + } + }, + { + "fold": 2, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.44417219953539205, + "rmse": 0.6664624517070651, + "mae": 0.5136315814618554, + "r2": 0.12276783858786378 + }, + "delivery": { + "n_samples": 60, + "mse": 0.6732209168800264, + "rmse": 0.8205004063862652, + "mae": 0.5596807344001717, + "r2": 0.10382628141775896 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9180327868852459, + "precision": 0.6875, + "recall": 0.9568965517241379, + "f1": 0.7502047502047502 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.75, + "precision": 0.6652173913043478, + "recall": 0.625366568914956, + "f1": 0.636063544460491 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7142857142857143, + "precision": 0.6787755102040816, + "recall": 0.6622315592903828, + "f1": 0.6671936758893281 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.3705731928348541, + "js_divergence": 0.0945676639676094 + } + } + }, + { + "fold": 3, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.9476472096853905, + "rmse": 1.3955813160419535, + "mae": 0.6786922237201852, + "r2": -0.10160416549494755 + }, + "delivery": { + "n_samples": 59, + "mse": 1.0546120682688678, + "rmse": 1.0269430696337882, + "mae": 0.7278208352972643, + "r2": -0.025208277515615496 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9166666666666666, + "precision": 0.6875, + "recall": 0.956140350877193, + "f1": 0.749791492910759 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7738095238095238, + "precision": 0.7091007583965331, + "recall": 0.6414956011730205, + "f1": 0.6571428571428573 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6071428571428571, + "precision": 0.4344751866490997, + "recall": 0.443614881850176, + "f1": 0.4324842382393017 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.9395005106925964, + "js_divergence": 0.12415548413991928 + } + } + }, + { + "fold": 4, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.7430198511001034, + "rmse": 0.8619859924036488, + "mae": 0.5445426530481893, + "r2": 0.005747196126379084 + }, + "delivery": { + "n_samples": 58, + "mse": 1.0484304054163862, + "rmse": 1.0239289064268018, + "mae": 0.6575078364333202, + "r2": -0.013814952507488876 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.8659090909090909 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6666666666666666, + "precision": 0.5142857142857142, + "recall": 0.5102639296187683, + "f1": 0.5050505050505051 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6547619047619048, + "precision": 0.6187822362094589, + "recall": 0.6354700854700855, + "f1": 0.6152751423149906 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.4118382930755615, + "js_divergence": 0.1084776222705841 + } + } + } + ], + "summary_stats": { + "size": { + "mse_mean": 1.083135285216074, + "mse_std": 0.6406941255063764, + "rmse_mean": 0.9943008101240769, + "rmse_std": 0.3074104490785543, + "mae_mean": 0.5920610175206343, + "mae_std": 0.0794441259199241, + "r2_mean": -0.1268065213630016, + "r2_std": 0.2408707365476069 + }, + "delivery": { + "mse_mean": 0.9294118681418391, + "mse_std": 0.18272890550921916, + "rmse_mean": 0.9591394958074021, + "rmse_std": 0.09727947226502257, + "mae_mean": 0.6555577728181625, + "mae_std": 0.05564855198139901, + "r2_mean": 0.052994332876774665, + "r2_std": 0.06689360454000906 + }, + "toxic": { + "accuracy_mean": 0.9468758882575938, + "accuracy_std": 0.02614298041760053, + "precision_mean": 0.7832456140350876, + "precision_std": 0.11222934338016616, + "recall_mean": 0.9238635381557341, + "recall_std": 0.08750485258067292, + "f1_mean": 0.8029745771294039, + "f1_std": 0.04590925112738489 + }, + "pdi": { + "accuracy_mean": 0.6857142857142857, + "accuracy_std": 0.09391944249198107, + "precision_mean": 0.5949826775592237, + "precision_std": 0.08721131763365055, + "recall_mean": 0.5661663755150064, + "recall_std": 0.06466900236745027, + "f1_mean": 0.5695376582109668, + "f1_std": 0.07421092597308579 + }, + "ee": { + "accuracy_mean": 0.6595238095238095, + "accuracy_std": 0.053558199424771406, + "precision_mean": 0.5887923967402736, + "precision_std": 0.09316564987347752, + "recall_mean": 0.5856421169866548, + "recall_std": 0.08436676188679884, + "f1_mean": 0.5765347130990797, + "f1_std": 0.08624597369019231 + }, + "biodist": { + "kl_divergence_mean": 0.5694257736206054, + "kl_divergence_std": 0.24356696856764282, + "js_divergence_mean": 0.10754408240318299, + "js_divergence_std": 0.016673227855747277 + } + } +} \ No newline at end of file diff --git a/models/abl_full/tanimoto_knn_combined/seed42/outer_fold_0/test_metrics.json b/models/abl_full/tanimoto_knn_combined/seed42/outer_fold_0/test_metrics.json new file mode 100644 index 0000000..7f544b1 --- /dev/null +++ b/models/abl_full/tanimoto_knn_combined/seed42/outer_fold_0/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 83, + "mse": 1.5140121527873271, + "rmse": 1.230452011574335, + "mae": 0.5724796654589205, + "r2": 0.07338730189412301 + }, + "delivery": { + "n_samples": 58, + "mse": 0.5476637789504636, + "rmse": 0.7400430926307356, + "mae": 0.510017358783757, + "r2": 0.3038862836718136 + }, + "toxic": { + "n_samples": 58, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7738095238095238, + "precision": 0.7291666666666666, + "recall": 0.6411261582323593, + "f1": 0.6571428571428571 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.75, + "precision": 0.6861888111888113, + "recall": 0.6571428571428571, + "f1": 0.6684303350970019 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.1652657687664032, + "js_divergence": 0.036902397871017456 + } +} \ No newline at end of file diff --git a/models/abl_full/tanimoto_knn_combined/seed42/outer_fold_1/test_metrics.json b/models/abl_full/tanimoto_knn_combined/seed42/outer_fold_1/test_metrics.json new file mode 100644 index 0000000..6d2b2f0 --- /dev/null +++ b/models/abl_full/tanimoto_knn_combined/seed42/outer_fold_1/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.32749766472170727, + "rmse": 0.5722741167672248, + "mae": 0.4118617074231484, + "r2": -6.143273825820472e-05 + }, + "delivery": { + "n_samples": 61, + "mse": 0.8565509328550216, + "rmse": 0.9255003689113374, + "mae": 0.5844964376177455, + "r2": 0.35165271085214567 + }, + "toxic": { + "n_samples": 61, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7380952380952381, + "precision": 0.6382791922739245, + "recall": 0.6190476190476191, + "f1": 0.6259109311740891 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6666666666666666, + "precision": 0.5905741084312512, + "recall": 0.5836782265353694, + "f1": 0.5857980873508823 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.17308977246284485, + "js_divergence": 0.04224603995680809 + } +} \ No newline at end of file diff --git a/models/abl_full/tanimoto_knn_combined/seed42/outer_fold_2/test_metrics.json b/models/abl_full/tanimoto_knn_combined/seed42/outer_fold_2/test_metrics.json new file mode 100644 index 0000000..0566cd6 --- /dev/null +++ b/models/abl_full/tanimoto_knn_combined/seed42/outer_fold_2/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.42245877771638074, + "rmse": 0.6499682897775712, + "mae": 0.4708858553092882, + "r2": 0.1656514588906851 + }, + "delivery": { + "n_samples": 60, + "mse": 0.6954245815400071, + "rmse": 0.8339212082325327, + "mae": 0.5157280358097827, + "r2": 0.07426935556241732 + }, + "toxic": { + "n_samples": 61, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7619047619047619, + "precision": 0.6920821114369502, + "recall": 0.6920821114369502, + "f1": 0.6920821114369502 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.5927101200686107, + "recall": 0.5814659197012139, + "f1": 0.5866098376702831 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.11873765289783478, + "js_divergence": 0.02909410372376442 + } +} \ No newline at end of file diff --git a/models/abl_full/tanimoto_knn_combined/seed42/outer_fold_3/test_metrics.json b/models/abl_full/tanimoto_knn_combined/seed42/outer_fold_3/test_metrics.json new file mode 100644 index 0000000..31deac2 --- /dev/null +++ b/models/abl_full/tanimoto_knn_combined/seed42/outer_fold_3/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 83, + "mse": 1.7192147565813247, + "rmse": 1.3111882994373176, + "mae": 0.5811085510825043, + "r2": 0.02759897798108102 + }, + "delivery": { + "n_samples": 59, + "mse": 0.8146305386985778, + "rmse": 0.9025688553781245, + "mae": 0.5916744657965788, + "r2": 0.20808229251377597 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9666666666666667, + "precision": 0.8, + "recall": 0.9824561403508771, + "f1": 0.8660714285714286 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7261904761904762, + "precision": 0.6246376811594203, + "recall": 0.594574780058651, + "f1": 0.6014029296472045 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6309523809523809, + "precision": 0.486324096493588, + "recall": 0.4656611362493715, + "f1": 0.4676236044657098 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.1976311206817627, + "js_divergence": 0.04640261083841324 + } +} \ No newline at end of file diff --git a/models/abl_full/tanimoto_knn_combined/seed42/outer_fold_4/test_metrics.json b/models/abl_full/tanimoto_knn_combined/seed42/outer_fold_4/test_metrics.json new file mode 100644 index 0000000..2d95ae5 --- /dev/null +++ b/models/abl_full/tanimoto_knn_combined/seed42/outer_fold_4/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.6080370657345592, + "rmse": 0.7797673151232739, + "mae": 0.4776152567917182, + "r2": 0.18637092054727933 + }, + "delivery": { + "n_samples": 58, + "mse": 0.9823878715939768, + "rmse": 0.9911548171673166, + "mae": 0.5694479240778962, + "r2": 0.05004709111957395 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9830508474576272, + "precision": 0.875, + "recall": 0.9910714285714286, + "f1": 0.9240669240669239 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7738095238095238, + "precision": 0.7045654082528534, + "recall": 0.6708211143695014, + "f1": 0.6827668455575433 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6666666666666666, + "precision": 0.5982173382173382, + "recall": 0.6111111111111112, + "f1": 0.6041702534672405 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.17170464992523193, + "js_divergence": 0.043072134256362915 + } +} \ No newline at end of file diff --git a/models/abl_full/tanimoto_knn_combined/seed42/summary.json b/models/abl_full/tanimoto_knn_combined/seed42/summary.json new file mode 100644 index 0000000..bf6fde4 --- /dev/null +++ b/models/abl_full/tanimoto_knn_combined/seed42/summary.json @@ -0,0 +1,287 @@ +{ + "fold_results": [ + { + "fold": 0, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.5140121527873271, + "rmse": 1.230452011574335, + "mae": 0.5724796654589205, + "r2": 0.07338730189412301 + }, + "delivery": { + "n_samples": 58, + "mse": 0.5476637789504636, + "rmse": 0.7400430926307356, + "mae": 0.510017358783757, + "r2": 0.3038862836718136 + }, + "toxic": { + "n_samples": 58, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7738095238095238, + "precision": 0.7291666666666666, + "recall": 0.6411261582323593, + "f1": 0.6571428571428571 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.75, + "precision": 0.6861888111888113, + "recall": 0.6571428571428571, + "f1": 0.6684303350970019 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.1652657687664032, + "js_divergence": 0.036902397871017456 + } + } + }, + { + "fold": 1, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.32749766472170727, + "rmse": 0.5722741167672248, + "mae": 0.4118617074231484, + "r2": -6.143273825820472e-05 + }, + "delivery": { + "n_samples": 61, + "mse": 0.8565509328550216, + "rmse": 0.9255003689113374, + "mae": 0.5844964376177455, + "r2": 0.35165271085214567 + }, + "toxic": { + "n_samples": 61, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7380952380952381, + "precision": 0.6382791922739245, + "recall": 0.6190476190476191, + "f1": 0.6259109311740891 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6666666666666666, + "precision": 0.5905741084312512, + "recall": 0.5836782265353694, + "f1": 0.5857980873508823 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.17308977246284485, + "js_divergence": 0.04224603995680809 + } + } + }, + { + "fold": 2, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.42245877771638074, + "rmse": 0.6499682897775712, + "mae": 0.4708858553092882, + "r2": 0.1656514588906851 + }, + "delivery": { + "n_samples": 60, + "mse": 0.6954245815400071, + "rmse": 0.8339212082325327, + "mae": 0.5157280358097827, + "r2": 0.07426935556241732 + }, + "toxic": { + "n_samples": 61, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7619047619047619, + "precision": 0.6920821114369502, + "recall": 0.6920821114369502, + "f1": 0.6920821114369502 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.5927101200686107, + "recall": 0.5814659197012139, + "f1": 0.5866098376702831 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.11873765289783478, + "js_divergence": 0.02909410372376442 + } + } + }, + { + "fold": 3, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.7192147565813247, + "rmse": 1.3111882994373176, + "mae": 0.5811085510825043, + "r2": 0.02759897798108102 + }, + "delivery": { + "n_samples": 59, + "mse": 0.8146305386985778, + "rmse": 0.9025688553781245, + "mae": 0.5916744657965788, + "r2": 0.20808229251377597 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9666666666666667, + "precision": 0.8, + "recall": 0.9824561403508771, + "f1": 0.8660714285714286 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7261904761904762, + "precision": 0.6246376811594203, + "recall": 0.594574780058651, + "f1": 0.6014029296472045 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6309523809523809, + "precision": 0.486324096493588, + "recall": 0.4656611362493715, + "f1": 0.4676236044657098 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.1976311206817627, + "js_divergence": 0.04640261083841324 + } + } + }, + { + "fold": 4, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.6080370657345592, + "rmse": 0.7797673151232739, + "mae": 0.4776152567917182, + "r2": 0.18637092054727933 + }, + "delivery": { + "n_samples": 58, + "mse": 0.9823878715939768, + "rmse": 0.9911548171673166, + "mae": 0.5694479240778962, + "r2": 0.05004709111957395 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9830508474576272, + "precision": 0.875, + "recall": 0.9910714285714286, + "f1": 0.9240669240669239 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7738095238095238, + "precision": 0.7045654082528534, + "recall": 0.6708211143695014, + "f1": 0.6827668455575433 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6666666666666666, + "precision": 0.5982173382173382, + "recall": 0.6111111111111112, + "f1": 0.6041702534672405 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.17170464992523193, + "js_divergence": 0.043072134256362915 + } + } + } + ], + "summary_stats": { + "size": { + "mse_mean": 0.9182440835082598, + "mse_std": 0.5809486046321074, + "rmse_mean": 0.9087300065359445, + "rmse_std": 0.3040622612713425, + "mae_mean": 0.502790207213116, + "mae_std": 0.06466844895343435, + "r2_mean": 0.09058944531498206, + "r2_std": 0.07387786039775746 + }, + "delivery": { + "mse_mean": 0.7793315407276095, + "mse_std": 0.14775050384356425, + "rmse_mean": 0.8786376684640093, + "rmse_std": 0.08560016520859612, + "mae_mean": 0.5542728444171521, + "mae_std": 0.03460312179472036, + "r2_mean": 0.1975875467439453, + "r2_std": 0.1201009560953211 + }, + "toxic": { + "accuracy_mean": 0.9899435028248588, + "accuracy_std": 0.01336202982968992, + "precision_mean": 0.9349999999999999, + "precision_std": 0.08306623862918074, + "recall_mean": 0.9947055137844611, + "recall_std": 0.007033469665969538, + "f1_mean": 0.9580276705276706, + "f1_std": 0.05457895575931689 + }, + "pdi": { + "accuracy_mean": 0.7547619047619047, + "accuracy_std": 0.01934294858246658, + "precision_mean": 0.6777462119579629, + "precision_std": 0.03986731029940538, + "recall_mean": 0.6435303566290163, + "recall_std": 0.03493963777638951, + "f1_mean": 0.6518611349917288, + "f1_std": 0.03411811848690924 + }, + "ee": { + "accuracy_mean": 0.6785714285714285, + "accuracy_std": 0.03912303982179759, + "precision_mean": 0.5908028948799198, + "precision_std": 0.06336080202065292, + "recall_mean": 0.5798118501479846, + "recall_std": 0.0632578827905231, + "f1_mean": 0.5825264236102234, + "f1_std": 0.06492761089241221 + }, + "biodist": { + "kl_divergence_mean": 0.1652857929468155, + "kl_divergence_std": 0.025748995283173436, + "js_divergence_mean": 0.039543457329273224, + "js_divergence_std": 0.006049259234176996 + } + } +} \ No newline at end of file diff --git a/scripts/precompute_mole.py b/scripts/precompute_mole.py new file mode 100644 index 0000000..f8e84d1 --- /dev/null +++ b/scripts/precompute_mole.py @@ -0,0 +1,59 @@ +"""在 mole 环境中运行：把数据集所有唯一 SMILES 编码成 MolE 表征缓存。 + +用法（工作目录 = lnp_ml 根，且已 pip install -e mole_public）: + pyenv activate mole + python scripts/precompute_mole.py --ckpt MolE_ckpt/model.pth \ + --out data/processed/mole_embeddings.npz + +产物 .npz 含 "smiles" (N,) 与 "embeddings" (N, D=8000)。 +""" +import argparse +from pathlib import Path +import numpy as np +import pandas as pd + +DATA_FILES = ["data/interim/internal.csv", "data/external/all_data_LiON.csv"] +GLOB_FILES = ["data/external/all_amine_split_for_LiON/cv_*/train.csv", + "data/external/all_amine_split_for_LiON/cv_*/test.csv"] + +def collect_smiles(root: Path) -> list: + seen = set() + files = [root / f for f in DATA_FILES] + for pat in GLOB_FILES: + files += sorted(root.glob(pat)) + for f in files: + if not f.exists(): + print(f"跳过不存在: {f}"); continue + df = pd.read_csv(f, low_memory=False) + if "smiles" in df.columns: + seen.update(df["smiles"].dropna().astype(str).tolist()) + return sorted(seen) + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--ckpt", default="MolE_ckpt/model.pth", help="MolE 权重(或权重目录)") + ap.add_argument("--out", default="data/processed/mole_embeddings.npz") + ap.add_argument("--root", default=".") + ap.add_argument("--batch-size", type=int, default=32) + ap.add_argument("--num-workers", type=int, default=4) + args = ap.parse_args() + + from mole import mole_predict + + smiles = collect_smiles(Path(args.root)) + print(f"收集到 {len(smiles)} 个唯一 SMILES，开始 MolE 编码...") + + emb = mole_predict.encode( + smiles=smiles, pretrained_model=args.ckpt, + batch_size=args.batch_size, num_workers=args.num_workers, + ) + emb = np.asarray(emb, dtype=np.float32) + print(f"embeddings.shape = {emb.shape}") # 期望 (N, 8000) + assert emb.shape[0] == len(smiles), "encode 丢弃了部分 SMILES，需对齐后再存" + + out = Path(args.out); out.parent.mkdir(parents=True, exist_ok=True) + np.savez_compressed(out, smiles=np.array(smiles), embeddings=emb) + print(f"已保存 {emb.shape} 到 {out}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/precompute_moleculestm.py b/scripts/precompute_moleculestm.py new file mode 100644 index 0000000..e410ff0 --- /dev/null +++ b/scripts/precompute_moleculestm.py @@ -0,0 +1,118 @@ +"""在 moleculestm 环境中运行：把数据集里所有唯一 SMILES 编码成 MoleculeSTM(Graph 版) 表征缓存。 + +用法（工作目录 = lnp_ml 根）: + conda activate moleculestm + export HF_HUB_OFFLINE=1 # 只用本地权重 + python scripts/precompute_moleculestm.py \ + --stm-repo /path/to/MoleculeSTM \ + --ckpt MoleculeSTM_ckpt/pretrained_MoleculeSTM/<某个Graph变体>/molecule_model.pth \ + --out data/processed/moleculestm_embeddings.npz + +产物 .npz 含 "smiles" (N,) 与 "embeddings" (N, D=300)，供主项目查表使用。 +""" +import argparse +import sys +from pathlib import Path + +import numpy as np +import pandas as pd +import torch + +DATA_FILES = [ + "data/interim/internal.csv", + "data/external/all_data_LiON.csv", +] +GLOB_FILES = [ + "data/external/all_amine_split_for_LiON/cv_*/train.csv", + "data/external/all_amine_split_for_LiON/cv_*/test.csv", +] + + +def collect_smiles(root: Path) -> list: + seen: set = set() + files = [root / f for f in DATA_FILES] + for pattern in GLOB_FILES: + files += sorted(root.glob(pattern)) + for f in files: + if not f.exists(): + print(f"跳过不存在的文件: {f}") + continue + df = pd.read_csv(f, low_memory=False) + if "smiles" in df.columns: + seen.update(df["smiles"].dropna().astype(str).tolist()) + return sorted(seen) + + +def build_encoder(stm_repo: str, ckpt: str, device: str): + """加载 MoleculeSTM Graph 分子编码器（GNN_graphpred 包装的 GIN）。""" + sys.path.insert(0, stm_repo) + from MoleculeSTM.models import GNN, GNN_graphpred # 官方模型定义 + + node = GNN(num_layer=5, emb_dim=300, JK="last", drop_ratio=0.0, gnn_type="gin") + model = GNN_graphpred( + num_layer=5, emb_dim=300, num_tasks=1, JK="last", + graph_pooling="mean", molecule_node_model=node, + ) + state = torch.load(ckpt, map_location="cpu") + model.load_state_dict(state) # 与官方一致，key 完全匹配 + model.eval().to(device) + return model + + +def smiles_to_data(smiles: str, stm_repo: str): + """SMILES -> torch_geometric Data（用官方 mol_to_graph_data_obj_simple）。""" + from rdkit import Chem + # 官方图特征化函数（不同版本路径可能是 datasets.utils 或 datasets.molecule_datasets） + from MoleculeSTM.datasets.utils import mol_to_graph_data_obj_simple + + mol = Chem.MolFromSmiles(smiles) + if mol is None: + return None + return mol_to_graph_data_obj_simple(mol) + + +@torch.no_grad() +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--stm-repo", required=True, help="官方 MoleculeSTM 仓库路径") + ap.add_argument("--ckpt", required=True, help="molecule_model.pth 路径") + ap.add_argument("--out", default="data/processed/moleculestm_embeddings.npz") + ap.add_argument("--root", default=".", help="lnp_ml 仓库根目录") + ap.add_argument("--batch-size", type=int, default=256) + ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu") + args = ap.parse_args() + + from torch_geometric.data import DataLoader + from torch_geometric.nn import global_mean_pool + + model = build_encoder(args.stm_repo, args.ckpt, args.device) + smiles = collect_smiles(Path(args.root)) + print(f"收集到 {len(smiles)} 个唯一 SMILES，开始编码...") + + # 图特征化（记录成功的 SMILES，跳过 RDKit 解析失败的） + data_list, kept = [], [] + for s in smiles: + d = smiles_to_data(s, args.stm_repo) + if d is not None: + data_list.append(d) + kept.append(s) + if len(kept) < len(smiles): + print(f"警告: {len(smiles) - len(kept)} 个 SMILES 无法图特征化，已跳过") + + loader = DataLoader(data_list, batch_size=args.batch_size, shuffle=False) + chunks = [] + for batch in loader: + batch = batch.to(args.device) + graph_repr, _ = model(batch.x, batch.edge_index, batch.edge_attr, batch.batch) # [B, 300] + chunks.append(graph_repr.cpu().numpy().astype(np.float32)) + print(f" 已编码 {sum(c.shape[0] for c in chunks)}/{len(kept)}") + embeddings = np.vstack(chunks) + + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + np.savez_compressed(out, smiles=np.array(kept), embeddings=embeddings) + print(f"已保存 {embeddings.shape} 到 {out}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/run_classical_baselines.py b/scripts/run_classical_baselines.py new file mode 100644 index 0000000..5652b5c --- /dev/null +++ b/scripts/run_classical_baselines.py @@ -0,0 +1,206 @@ +"""经典基线：RandomForest / Tanimoto-kNN / Tanimoto+tabular-kNN。 + +用法: + python scripts/run_classical_baselines.py \ + --input data/interim/internal.csv \ + --ref-run models/abl_full/s1_baseline/seed42 \ + --seed 42 --n-outer 5 --n-inner 3 --out-root models/abl_full +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +import pandas as pd +from scipy.special import rel_entr +from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier +from sklearn.model_selection import KFold +from sklearn.preprocessing import StandardScaler +from sklearn.metrics import (mean_squared_error, mean_absolute_error, r2_score, + accuracy_score, precision_score, recall_score, f1_score) + +from lnp_ml.dataset import LNPDataset +from lnp_ml.featurization.smiles import RDKitFeaturizer + +SEED = 42 +X_RF = MORGAN = TAB = None # 全局特征，main 里填充 + +# ---------- 指标（对齐 evaluate_on_test）---------- +def reg_metrics(t, p): + t, p = np.asarray(t), np.asarray(p) + return {"n_samples": int(len(p)), "mse": float(mean_squared_error(t, p)), + "rmse": float(np.sqrt(mean_squared_error(t, p))), + "mae": float(mean_absolute_error(t, p)), "r2": float(r2_score(t, p))} + +def clf_metrics(t, p): + return {"n_samples": int(len(p)), "accuracy": float(accuracy_score(t, p)), + "precision": float(precision_score(t, p, average="macro", zero_division=0)), + "recall": float(recall_score(t, p, average="macro", zero_division=0)), + "f1": float(f1_score(t, p, average="macro", zero_division=0))} + +def dist_metrics(t, p, eps=1e-10): + t = np.clip(np.asarray(t), eps, 1.0); p = np.clip(np.asarray(p), eps, 1.0) + kl = float(np.sum(rel_entr(t, p), axis=-1).mean()) + m = 0.5 * (t + p) + js = float((0.5*np.sum(rel_entr(t, m), axis=-1) + 0.5*np.sum(rel_entr(p, m), axis=-1)).mean()) + return {"n_samples": int(len(p)), "kl_divergence": kl, "js_divergence": js} + +METRICS = {"reg": reg_metrics, "clf": clf_metrics, "dist": dist_metrics} + +def score(task_type, t, p): + if task_type == "reg": return r2_score(t, p) + if task_type == "clf": return f1_score(t, p, average="macro", zero_division=0) + return -dist_metrics(t, p)["js_divergence"] + +# ---------- 相似度 / 距离 ---------- +def tanimoto_sim(A, B): + inter = A @ B.T + a = A.sum(1)[:, None]; b = B.sum(1)[None, :] + return inter / np.clip(a + b - inter, 1e-10, None) + +def combined_dist(fp_te, fp_tr, tab_te, tab_tr, alpha): + d_fp = 1.0 - tanimoto_sim(fp_te, fp_tr) + sc = StandardScaler().fit(tab_tr) + Ztr, Zte = sc.transform(tab_tr), sc.transform(tab_te) + d_tab = np.sqrt(np.clip(((Zte[:, None, :] - Ztr[None, :, :]) ** 2).sum(-1), 0, None)) + def mm(d): + lo, hi = float(d.min()), float(d.max()) + return (d - lo) / (hi - lo) if hi > lo else np.zeros_like(d) + return alpha * mm(d_fp) + (1 - alpha) * mm(d_tab) + +def knn_from_matrix(S, is_sim, ytr, k, task_type, n_classes): + k = min(k, S.shape[1]) + order = np.argsort(-S if is_sim else S, axis=1)[:, :k] + out = [] + for i, idx in enumerate(order): + w = S[i, idx] if is_sim else 1.0 / (S[i, idx] + 1e-6) + w = np.clip(w, 1e-12, None) + yy = ytr[idx] + if task_type == "reg": + out.append(float(np.average(yy, weights=w))) + elif task_type == "clf": + sc = np.zeros(n_classes) + for c, wt in zip(yy, w): sc[int(c)] += wt + out.append(int(sc.argmax())) + else: + v = np.average(yy, axis=0, weights=w); s = v.sum() + out.append(v / s if s > 0 else v) + return np.array(out) + +# ---------- 各模型预测 ---------- +RF_GRID = [dict(n_estimators=300, max_depth=None), + dict(n_estimators=300, max_depth=12), + dict(n_estimators=600, max_depth=None)] +K_GRID = [3, 5, 10, 15] +ALPHA_GRID = [0.3, 0.5, 0.7] + +def rf_predict(tr, te, y, task_type, nc, params): + Xtr, Xte, ytr = X_RF[tr], X_RF[te], y[tr] + if task_type == "reg": + m = RandomForestRegressor(random_state=SEED, n_jobs=-1, **params); m.fit(Xtr, ytr); return m.predict(Xte) + if task_type == "clf": + m = RandomForestClassifier(random_state=SEED, n_jobs=-1, **params); m.fit(Xtr, ytr); return m.predict(Xte) + m = RandomForestRegressor(random_state=SEED, n_jobs=-1, **params); m.fit(Xtr, ytr) + v = np.clip(m.predict(Xte), 0, None); s = v.sum(1, keepdims=True); return np.where(s > 0, v / s, v) + +def knn_predict(variant, tr, te, y, task_type, nc, params): + if variant == "tanimoto_knn": + S, is_sim = tanimoto_sim(MORGAN[te], MORGAN[tr]), True + else: + S, is_sim = combined_dist(MORGAN[te], MORGAN[tr], TAB[te], TAB[tr], params.get("alpha", 0.5)), False + return knn_from_matrix(S, is_sim, y[tr], params["k"], task_type, nc) + +def grid_for(model): + if model == "rf": return [{"rf": g} for g in RF_GRID] + if model == "tanimoto_knn": return [{"k": k} for k in K_GRID] + return [{"k": k, "alpha": a} for k in K_GRID for a in ALPHA_GRID] + +def predict(model, tr, te, y, task_type, nc, params): + return rf_predict(tr, te, y, task_type, nc, params["rf"]) if model == "rf" \ + else knn_predict(model, tr, te, y, task_type, nc, params) + +def select_params(model, tr, y, valid, task_type, nc, n_inner): + cand = grid_for(model) + if len(cand) == 1: return cand[0] + n_splits = max(2, min(n_inner, len(tr) // 2)) + kf = KFold(n_splits=n_splits, shuffle=True, random_state=SEED) + best, best_s = cand[0], -1e18 + for p in cand: + scs = [] + for itr, iva in kf.split(tr): + a, b = tr[itr], tr[iva] + a, b = a[valid[a]], b[valid[b]] + if len(a) < 2 or len(b) < 1: continue + try: + scs.append(score(task_type, y[b], predict(model, a, b, y, task_type, nc, p))) + except Exception: + pass + if scs and float(np.mean(scs)) > best_s: + best_s, best = float(np.mean(scs)), p + return best + +def main(): + global SEED, X_RF, MORGAN, TAB + ap = argparse.ArgumentParser() + ap.add_argument("--input", default="data/interim/internal.csv") + ap.add_argument("--ref-run", default="models/abl_full/s1_baseline/seed42") + ap.add_argument("--seed", type=int, default=42) + ap.add_argument("--n-outer", type=int, default=5) + ap.add_argument("--n-inner", type=int, default=3) + ap.add_argument("--out-root", default="models/abl_full") + ap.add_argument("--models", nargs="+", + default=["rf", "tanimoto_knn", "tanimoto_knn_combined"]) + args = ap.parse_args() + SEED = args.seed + + ds = LNPDataset(pd.read_csv(args.input)) + MORGAN = RDKitFeaturizer().transform(ds.smiles)["morgan"].astype(np.float32) + TAB = np.concatenate([ds.comp, ds.phys, ds.help, ds.exp], axis=1).astype(np.float32) + X_RF = np.concatenate([MORGAN, TAB], axis=1).astype(np.float32) + + tasks = {} + def add(name, ttype, y, valid, nc=None): + if y is not None: + tasks[name] = (ttype, np.asarray(y), np.asarray(valid), nc) + add("size", "reg", ds.size, ~np.isnan(ds.size) if ds.size is not None else None) + add("delivery", "reg", ds.delivery, ~np.isnan(ds.delivery) if ds.delivery is not None else None) + if ds.toxic is not None: add("toxic", "clf", ds.toxic, ds.toxic >= 0, 2) + if ds.pdi is not None: add("pdi", "clf", ds.pdi, ds.pdi_valid, int(ds.pdi[ds.pdi_valid].max()) + 1) + if ds.ee is not None: add("ee", "clf", ds.ee, ds.ee_valid, int(ds.ee[ds.ee_valid].max()) + 1) + if ds.biodist is not None: add("biodist", "dist", ds.biodist, ds.biodist_valid) + + folds = [] + for k in range(args.n_outer): + d = json.loads((Path(args.ref_run) / f"outer_fold_{k}" / "splits.json").read_text()) + folds.append((np.array(d["outer_train_idx"]), np.array(d["outer_test_idx"]))) + + for model in args.models: + fold_results, agg = [], {} + for k, (tr, te) in enumerate(folds): + tm = {} + for name, (ttype, y, valid, nc) in tasks.items(): + trv, tev = tr[valid[tr]], te[valid[te]] + if len(trv) < 2 or len(tev) < 1: continue + params = select_params(model, tr, y, valid, ttype, nc, args.n_inner) + m = METRICS[ttype](y[tev], predict(model, trv, tev, y, ttype, nc, params)) + tm[name] = m + for mk, mv in m.items(): + if mk == "n_samples": continue + agg.setdefault(name, {}).setdefault(mk, []).append(mv) + fold_results.append({"fold": k, "test_metrics": tm}) + fdir = Path(args.out_root) / model / f"seed{args.seed}" / f"outer_fold_{k}" + fdir.mkdir(parents=True, exist_ok=True) + (fdir / "test_metrics.json").write_text(json.dumps(tm, indent=2)) + summary_stats = {} + for t, md in agg.items(): + summary_stats[t] = {} + for mk, v in md.items(): + summary_stats[t][f"{mk}_mean"] = float(np.mean(v)) + summary_stats[t][f"{mk}_std"] = float(np.std(v)) + run_dir = Path(args.out_root) / model / f"seed{args.seed}" + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "summary.json").write_text( + json.dumps({"fold_results": fold_results, "summary_stats": summary_stats}, indent=2)) + print(f"[{model}] saved -> {run_dir / 'summary.json'}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts_run/run_ablation_full.sh b/scripts_run/run_ablation_full.sh index 7ca853f..939b47e 100644 --- a/scripts_run/run_ablation_full.sh +++ b/scripts_run/run_ablation_full.sh @@ -99,10 +99,16 @@ echo "[$(date '+%F %T')] START seed=${SEED} pool=${GPU_POOL}" for g in "${POOL[@]}"; do worker "$g" & done wait +# ===== 经典基线（CPU，复用 s1_baseline 的 split）===== +python scripts/run_classical_baselines.py \ + --input "${INPUT}" --ref-run "models/abl_full/s1_baseline/seed${SEED}" \ + --seed ${SEED} --n-outer ${N_OUTER} --n-inner ${N_INNER} \ + --out-root models/abl_full 2>&1 | ts | tee logs/classical.log + # ===== 汇总 ===== latest(){ ls -dt "models/abl_full/$1/seed${SEED}"/summary.json "models/abl_full/$1/seed${SEED}"/*/summary.json 2>/dev/null | head -1 | xargs -r dirname; } sum_args=() -for name in s1_baseline s1_moe s1_llm s1_both s2_chemeleon s2_unimol s3_molt5 s3_biot5; do +for name in s1_baseline s1_moe s1_llm s1_both s2_chemeleon s2_unimol s2_moleculestm s2_mole s3_molt5 s3_biot5 rf tanimoto_knn tanimoto_knn_combined; do d=$(latest "$name"); [ -n "$d" ] && sum_args+=(--run "${name}=$d") done d=$(latest s1_baseline); [ -n "$d" ] && sum_args+=(--run "s2_mpnn=$d") diff --git a/scripts_run/run_both_chemeleon.sh b/scripts_run/run_both_chemeleon.sh new file mode 100644 index 0000000..7a174ed --- /dev/null +++ b/scripts_run/run_both_chemeleon.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -uo pipefail +cd "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +# ===== 环境（与原消融一致，保证可复现）===== +export TRANSFORMERS_OFFLINE=1 HF_HUB_OFFLINE=1 PYTHONUNBUFFERED=1 +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True TOKENIZERS_PARALLELISM=false +export PYTHONHASHSEED=${PYTHONHASHSEED:-42} + +# ===== 参数（与 s1_both 一致）===== +SEED=${SEED:-42} +GPU=${GPU:-0} +QBATCH=${QBATCH:-8} +N_OUTER=${N_OUTER:-5}; N_INNER=${N_INNER:-3} +N_TRIALS=${N_TRIALS:-20}; EPOCHS=${EPOCHS:-20}; PATIENCE=${PATIENCE:-5} +INPUT=${INPUT:-data/interim/internal.csv} +CHEM_CACHE=data/processed/chemeleon_embeddings.npz +QWEN=models/qwen2.5-7b-instruct + +SRC=models/abl_full/s1_both/seed${SEED} +DST=models/abl_full/s1_both_chemeleon/seed${SEED} + +# ===== 方案A：只拷 best_params + epoch_mean（切勿拷 test_metrics.json！）===== +for i in $(seq 0 $((N_OUTER-1))); do + mkdir -p "$DST/outer_fold_$i" + cp "$SRC/outer_fold_$i/best_params.json" "$DST/outer_fold_$i/" + cp "$SRC/outer_fold_$i/epoch_mean.json" "$DST/outer_fold_$i/" +done + +# ===== 自动续跑：崩溃/被抢占后重试；靠 --resume-dir 跳过已完成的 fold ===== +MAX_RETRY=${MAX_RETRY:-100} # 最多重试次数 +POLL=${POLL:-60} # 重试/等待间隔(秒) +NEED=${NEED:-14000} # 启动所需空闲显存(MiB)，Qwen QLoRA 约 14GB +mkdir -p logs + +wait_free(){ # 抢占期显存不够就等，避免一起来就 OOM + while :; do + free=$(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits -i "$GPU" 2>/dev/null) + [ "${free:-0}" -ge "$NEED" ] && return 0 + echo "[$(date '+%F %T')] GPU${GPU} 空闲 ${free}MiB (<${NEED})，等待 ${POLL}s..." + sleep "$POLL" + done +} + +n=1 +while :; do + wait_free + echo "[$(date '+%F %T')] >>> s1_both_chemeleon try $n on GPU${GPU}" + if CUDA_VISIBLE_DEVICES=$GPU python -m lnp_ml.modeling.nested_cv_optuna \ + --input-path "$INPUT" \ + --output-dir models/abl_full/s1_both_chemeleon --resume-dir "$DST" \ + --seed $SEED --device cuda \ + --n-outer-folds $N_OUTER --n-inner-folds $N_INNER \ + --n-trials $N_TRIALS --epochs-per-trial $EPOCHS --inner-patience $PATIENCE \ + --batch-size $QBATCH \ + --use-chemeleon --chemeleon-cache $CHEM_CACHE \ + --use-moe \ + --use-llm --use-rag --rag-top-k 4 --use-soft-prompt --no-llm-freeze \ + --llm-use-qlora --llm-model-path $QWEN; then + echo "[$(date '+%F %T')] <<< s1_both_chemeleon DONE"; break + fi + (( n >= MAX_RETRY )) && { echo "[$(date '+%F %T')] !!! FAILED x${MAX_RETRY}"; exit 1; } + echo "[$(date '+%F %T')] 崩溃/被抢占，${POLL}s 后断点续跑 (attempt $n)"; sleep "$POLL"; ((n++)) +done \ No newline at end of file diff --git a/scripts_run/run_mole.sh b/scripts_run/run_mole.sh new file mode 100644 index 0000000..38d99d6 --- /dev/null +++ b/scripts_run/run_mole.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -uo pipefail +cd ~/lnp_ml +export PYTHONUNBUFFERED=1 TOKENIZERS_PARALLELISM=false +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export PYTHONHASHSEED=${PYTHONHASHSEED:-42} + +GPU=${GPU:-0} +SEED=${SEED:-42} +NEED=${NEED:-8000} +POLL=${POLL:-60} +MAX_RETRY=${MAX_RETRY:-100} +NAME=s2_mole +RUNDIR=models/abl_full/${NAME}/seed${SEED} +mkdir -p "$RUNDIR" logs + +n=1 +while :; do + while :; do + free=$(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits -i "$GPU" 2>/dev/null) + [ "${free:-0}" -ge "$NEED" ] && break + echo "[$(date '+%F %T')] GPU${GPU} 剩 ${free}MiB(<${NEED}) 等 ${POLL}s..." + sleep "$POLL" + done + + echo "[$(date '+%F %T')] >>> try $n on GPU${GPU}" + if CUDA_VISIBLE_DEVICES="$GPU" python -m lnp_ml.modeling.nested_cv_optuna \ + --input-path data/interim/internal.csv \ + --output-dir models/abl_full/${NAME} --resume-dir "${RUNDIR}" \ + --seed ${SEED} --device cuda \ + --n-outer-folds 5 --n-inner-folds 3 \ + --n-trials 20 --epochs-per-trial 20 --inner-patience 5 \ + --batch-size 16 \ + --use-mole --mole-cache data/processed/mole_embeddings.npz \ + 2>&1 | tee -a "${RUNDIR}/run.log"; then + echo "[$(date '+%F %T')] <<< DONE"; break + fi + (( n > MAX_RETRY )) && { echo "FAILED x${MAX_RETRY}"; exit 1; } + echo "[$(date '+%F %T')] 崩溃/被抢占, ${POLL}s 后断点续跑 (try $((n+1)))..." + sleep "$POLL"; ((n++)) +done \ No newline at end of file diff --git a/scripts_run/run_moleculestm.sh b/scripts_run/run_moleculestm.sh new file mode 100644 index 0000000..0f92335 --- /dev/null +++ b/scripts_run/run_moleculestm.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -uo pipefail +cd ~/lnp_ml +export PYTHONUNBUFFERED=1 TOKENIZERS_PARALLELISM=false +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export PYTHONHASHSEED=${PYTHONHASHSEED:-42} + +GPU=${GPU:-0} # 用 GPU0 +SEED=${SEED:-42} +NEED=${NEED:-8000} # 需要的空闲显存(MiB) +POLL=${POLL:-60} # 轮询/重试间隔(秒) +MAX_RETRY=${MAX_RETRY:-100} +NAME=s2_moleculestm +RUNDIR=models/abl_full/${NAME}/seed${SEED} +mkdir -p "$RUNDIR" logs + +n=1 +while :; do + # 等显存(被抢占时会一直等,空出来再续跑) + while :; do + free=$(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits -i "$GPU" 2>/dev/null) + [ "${free:-0}" -ge "$NEED" ] && break + echo "[$(date '+%F %T')] GPU${GPU} 剩 ${free}MiB(<${NEED}) 等 ${POLL}s..." + sleep "$POLL" + done + + echo "[$(date '+%F %T')] >>> try $n on GPU${GPU}" + if CUDA_VISIBLE_DEVICES="$GPU" python -m lnp_ml.modeling.nested_cv_optuna \ + --input-path data/interim/internal.csv \ + --output-dir models/abl_full/${NAME} --resume-dir "${RUNDIR}" \ + --seed ${SEED} --device cuda \ + --n-outer-folds 5 --n-inner-folds 3 \ + --n-trials 20 --epochs-per-trial 20 --inner-patience 5 \ + --batch-size 16 \ + --use-moleculestm --moleculestm-cache data/processed/moleculestm_embeddings.npz \ + 2>&1 | tee -a "${RUNDIR}/run.log"; then + echo "[$(date '+%F %T')] <<< DONE"; break + fi + (( n > MAX_RETRY )) && { echo "FAILED x${MAX_RETRY}"; exit 1; } + echo "[$(date '+%F %T')] 崩溃/被抢占, ${POLL}s 后断点续跑 (try $((n+1)))..." + sleep "$POLL"; ((n++)) +done \ No newline at end of file diff --git a/tests/check_gates.py b/tests/check_gates.py new file mode 100644 index 0000000..c2cf8c8 --- /dev/null +++ b/tests/check_gates.py @@ -0,0 +1,44 @@ +import glob +import os + +import numpy as np +import torch + +BASE = "models/abl" +TASKS = ["size", "delivery", "pdi", "ee", "toxic", "biodist"] + + +def latest(v: str): + dirs = sorted(glob.glob(f"{BASE}/{v}/*/"), key=os.path.getmtime) + return dirs[-1] if dirs else None + + +for v in ["moe", "llm", "both"]: + run = latest(v) + if run is None: + print(f"\n=== {v}: 无 run 目录 ===") + continue + + files = sorted(glob.glob(os.path.join(run, "**", "model.pt"), recursive=True)) + print(f"\n=== {v} ({len(files)} folds) ===") + if not files: + print(" 未找到 model.pt（本次 run 未保存权重 -> 无法读取门控/log_vars）") + continue + + gm, gl = [], [] + for f in files: + sd = torch.load(f, map_location="cpu", weights_only=False)["model_state_dict"] + if "fusion.g_moe" in sd: + gm.append(sd["fusion.g_moe"].item()) + if "fusion.g_llm" in sd: + gl.append(sd["fusion.g_llm"].item()) + if gm: + print(f" g_moe: mean={np.mean(gm):+.4f} |abs|=[{min(map(abs, gm)):.4f},{max(map(abs, gm)):.4f}]") + if gl: + print(f" g_llm: mean={np.mean(gl):+.4f} |abs|=[{min(map(abs, gl)):.4f},{max(map(abs, gl)):.4f}]") + + sd = torch.load(files[0], map_location="cpu", weights_only=False)["model_state_dict"] + for t in TASKS: + lv = sd.get(f"head.log_vars.{t}") + if lv is not None: + print(f" {t:9s} log_var={lv.item():+.3f} eff_w={np.exp(-lv.item()):.3f}") \ No newline at end of file diff --git a/tests/compare_benchmark.py b/tests/compare_benchmark.py new file mode 100644 index 0000000..8d35524 --- /dev/null +++ b/tests/compare_benchmark.py @@ -0,0 +1,37 @@ +import json, glob, os +import numpy as np +from scipy.stats import ttest_rel, wilcoxon + +A_DIR = "models/benchmark_cmp/mpnn" +B_DIR = "models/benchmark_cmp/chemeleon" +# (metric_key, higher_better) +METRICS = [("best_val_rmse", False), ("best_val_r2", True), ("best_val_loss", False)] + + +def collect(base): + """返回 {(seed, fold_idx): {metric: value}}""" + out = {} + for f in glob.glob(os.path.join(base, "seed*", "cv_results.json")): + seed = os.path.basename(os.path.dirname(f)) + with open(f) as fh: + d = json.load(fh) + for r in d["fold_results"]: + out[(seed, r["fold_idx"])] = r + return out + + +a, b = collect(A_DIR), collect(B_DIR) +keys = sorted(set(a) & set(b)) +print(f"配对样本数 n = {len(keys)} (seed × fold)\n") +for m, hb in METRICS: + da = np.array([a[k][m] for k in keys]) + db = np.array([b[k][m] for k in keys]) + diff = (db - da) if hb else (da - db) # 正 = CheMeleon 更好 + tp = ttest_rel(db, da).pvalue + try: + wp = wilcoxon(db, da).pvalue + except ValueError: + wp = float("nan") + print(f"{m:16s} MPNN={da.mean():.4f}±{da.std():.4f} " + f"CheMeleon={db.mean():.4f}±{db.std():.4f} " + f"Δ(Chem优)={diff.mean():+.4f} Wilcoxon p={wp:.3f} t p={tp:.3f}") \ No newline at end of file diff --git a/tests/paired_test.py b/tests/paired_test.py new file mode 100644 index 0000000..ab9005f --- /dev/null +++ b/tests/paired_test.py @@ -0,0 +1,65 @@ +import json +import glob +import os + +import numpy as np +from scipy.stats import wilcoxon, ttest_rel + +BASE = "models/abl" + +# (task, metric, higher_better) +ROWS = [ + ("delivery", "r2", True), + ("delivery", "rmse", False), + ("biodist", "kl_divergence", False), + ("size", "r2", True), + ("size", "rmse", False), + ("pdi", "f1", True), + ("ee", "f1", True), + ("toxic", "f1", True), +] + + +def latest(v: str) -> str: + dirs = sorted(glob.glob(f"{BASE}/{v}/*/"), key=os.path.getmtime) + if not dirs: + raise FileNotFoundError(f"variant '{v}' 在 {BASE} 下没有 run 目录") + return dirs[-1] + + +def collect(run: str, task: str, metric: str) -> dict: + """按相对路径(fold/repeat)收集某任务某指标，便于跨变体配对。""" + out = {} + for f in glob.glob(os.path.join(run, "**", "test_metrics.json"), recursive=True): + with open(f) as fh: + d = json.load(fh) + if task in d and metric in d[task]: + out[os.path.relpath(f, run)] = d[task][metric] + return out + + +def paired(vA: str, vB: str, task: str, metric: str, higher_better: bool = True) -> None: + a, b = collect(latest(vA), task, metric), collect(latest(vB), task, metric) + keys = sorted(set(a) & set(b)) + if not keys: + print(f"{task}.{metric}: 无可配对数据") + return + da = np.array([a[k] for k in keys]) + db = np.array([b[k] for k in keys]) + diff = (db - da) if higher_better else (da - db) # 正 = B 更好 + tp = ttest_rel(db, da).pvalue + try: + wp = wilcoxon(db, da).pvalue + except ValueError: + wp = float("nan") + print( + f"{task + '.' + metric:18s} {vA}={da.mean():.4f} {vB}={db.mean():.4f} " + f"Δ(B优)={diff.mean():+.4f} n={len(keys)} Wilcoxon p={wp:.3f} t p={tp:.3f}" + ) + + +if __name__ == "__main__": + for vB in ["moe", "llm", "both"]: + print(f"\n===== baseline vs {vB} =====") + for task, metric, hb in ROWS: + paired("baseline", vB, task, metric, hb) \ No newline at end of file diff --git a/tests/test_fusion.py b/tests/test_fusion.py new file mode 100644 index 0000000..1d7e340 --- /dev/null +++ b/tests/test_fusion.py @@ -0,0 +1,105 @@ +"""ResidualConcatFusion 模块自检脚本 +""" + +from __future__ import annotations + +import sys +import traceback +from typing import Callable, List, Tuple + +import torch + +from lnp_ml.modeling.layers.fusion import ResidualConcatFusion + + +def test_full_tokens_shape() -> None: + fusion = ResidualConcatFusion(d_model=256) + chem = torch.randn(2, 4, 256) + tab = torch.randn(2, 4, 256) + f_moe = torch.randn(2, 256) + f_llm = torch.randn(2, 256) + out = fusion(chem, tab, f_moe, f_llm) + assert out.shape == (2, 256), f"out shape {out.shape} != (2, 256)" + assert fusion.fusion_dim == 256 + + +def test_without_extras() -> None: + """关闭 MoE 和 LLM 时只拼接 chem + tab。""" + fusion = ResidualConcatFusion(d_model=64) + chem = torch.randn(3, 4, 64) + tab = torch.randn(3, 4, 64) + assert fusion(chem, tab).shape == (3, 64) + + +def test_nchem3_with_moe() -> None: + fusion = ResidualConcatFusion(d_model=64) + chem = torch.randn(2, 3, 64) + tab = torch.randn(2, 4, 64) + f_moe = torch.randn(2, 64) + assert fusion(chem, tab, f_moe=f_moe).shape == (2, 64) + + +def test_attn_weights_shape() -> None: + """注意力只池化真实 token：序列长度 = 4 + 4 = 8；f_moe/f_llm 走门控残差，不参与 softmax。""" + fusion = ResidualConcatFusion(d_model=64) + chem = torch.randn(2, 4, 64) + tab = torch.randn(2, 4, 64) + f_moe = torch.randn(2, 64) + f_llm = torch.randn(2, 64) + out, weights = fusion(chem, tab, f_moe, f_llm, return_attn_weights=True) + assert out.shape == (2, 64) + assert weights.shape == (2, 8), f"weights shape {weights.shape} != (2, 8)" + + +def test_concat_strategy_rejected() -> None: + try: + ResidualConcatFusion(d_model=64, strategy="concat") + except ValueError: + return + raise AssertionError("concat 策略应被拒绝") + + +def test_grad_flows() -> None: + fusion = ResidualConcatFusion(d_model=32) + chem = torch.randn(2, 4, 32, requires_grad=True) + tab = torch.randn(2, 4, 32, requires_grad=True) + fusion(chem, tab).sum().backward() + assert chem.grad is not None and torch.isfinite(chem.grad).all() + + +# ────────────────────────────────────────────────────────────────────── +# Runner +# ────────────────────────────────────────────────────────────────────── + +def _collect_tests() -> List[Tuple[str, Callable[[], None]]]: + return [ + (name, fn) for name, fn in globals().items() + if name.startswith("test_") and callable(fn) + ] + + +def main() -> int: + tests = _collect_tests() + pass_count = 0 + fail: List[Tuple[str, str]] = [] + for name, fn in tests: + try: + fn() + except Exception: + fail.append((name, traceback.format_exc())) + print(f"[FAIL] {name}") + else: + pass_count += 1 + print(f"[ OK ] {name}") + print(f"\n{'='*60}") + print(f"Passed: {pass_count}/{len(tests)}") + if fail: + print(f"\n{'='*60}\nFailures:") + for name, tb in fail: + print(f"\n--- {name} ---\n{tb}") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/tests/test_llm_prompt.py b/tests/test_llm_prompt.py new file mode 100644 index 0000000..3293dd2 --- /dev/null +++ b/tests/test_llm_prompt.py @@ -0,0 +1,104 @@ +"""LLM prompt 模块自检脚本 + +需先下载 MolT5 权重，默认从 models/molt5-base 读取， +可用环境变量 MOLT5_PATH 覆盖。若权重缺失则跳过（不计失败）。 +""" + +from __future__ import annotations + +import os +import sys +import traceback +from typing import Callable, List, Tuple + +import torch + +from lnp_ml.modeling.layers.llm_prompt import LLMPromptEncoder + +MOLT5_PATH = os.environ.get("MOLT5_PATH", "models/molt5-base") +SMILES = ["CCO", "c1ccccc1"] + + +def _build(d_model: int = 256, **kw) -> LLMPromptEncoder: + return LLMPromptEncoder(d_model=d_model, model_name_or_path=MOLT5_PATH, **kw) + + +def test_output_shape() -> None: + enc = _build().eval() + out = enc(SMILES) + assert out.shape == (2, 256), f"out shape {out.shape} != (2, 256)" + + +def test_single_example() -> None: + enc = _build(d_model=128).eval() + out = enc(["CCO"]) + assert out.shape == (1, 128), f"out shape {out.shape} != (1, 128)" + + +def test_encoder_frozen() -> None: + enc = _build() + n_trainable = sum(int(p.requires_grad) for p in enc.encoder.parameters()) + assert n_trainable == 0, "默认应冻结 MolT5 encoder" + assert all(p.requires_grad for p in enc.proj_down.parameters()), "proj_down 应可训练" + + +def test_cache_used() -> None: + enc = _build().eval() + enc(SMILES) + assert all(s in enc._cache for s in SMILES), "冻结模式应缓存句向量" + # 第二次前向应复用缓存（不报错且形状一致） + assert enc(SMILES).shape == (2, 256) + + +def test_grad_flows_to_proj() -> None: + """冻结 encoder 时，梯度应能流到可训练的 proj_down。""" + enc = _build() + enc(SMILES).sum().backward() + g = enc.proj_down[0].weight.grad + assert g is not None and torch.isfinite(g).all(), "proj_down 未收到有效梯度" + + +# ────────────────────────────────────────────────────────────────────── +# Runner +# ────────────────────────────────────────────────────────────────────── + +def _collect_tests() -> List[Tuple[str, Callable[[], None]]]: + return [ + (name, fn) for name, fn in globals().items() + if name.startswith("test_") and callable(fn) + ] + + +def main() -> int: + try: + _build() + except Exception as e: + print(f"[SKIP] 无法加载 MolT5（path={MOLT5_PATH}）：{e}") + print(" 请先下载权重，或设置 MOLT5_PATH 后重试。") + return 0 + + tests = _collect_tests() + pass_count = 0 + fail: List[Tuple[str, str]] = [] + for name, fn in tests: + try: + fn() + except Exception: + fail.append((name, traceback.format_exc())) + print(f"[FAIL] {name}") + else: + pass_count += 1 + print(f"[ OK ] {name}") + + print(f"\n{'='*60}") + print(f"Passed: {pass_count}/{len(tests)}") + if fail: + print(f"\n{'='*60}\nFailures:") + for name, tb in fail: + print(f"\n--- {name} ---\n{tb}") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 0000000..e609697 --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,129 @@ +"""LNPModel 整图自检脚本 + +base/+moe 路径只需 rdkit + torch；+llm 路径需 MolT5。 +""" + +from __future__ import annotations + +import os +import sys +import traceback +from typing import Callable, Dict, List, Tuple + +import torch + +from lnp_ml.featurization.smiles import RDKitFeaturizer +from lnp_ml.modeling.models import LNPModelWithoutMPNN + +MOLT5_PATH = os.environ.get("MOLT5_PATH", "models/molt5-base") + +# 期望的任务输出维度 +EXPECTED = {"size": 1, "pdi": 2, "ee": 3, "delivery": 1, "biodist": 7, "toxic": 2} + + +def _input_dims() -> Dict[str, int]: + """根据实际 RDKit 输出确定无 MPNN 时的输入维度。""" + feat = RDKitFeaturizer().transform(["CCO"]) + return { + "morgan": feat["morgan"].shape[1], + "maccs": feat["maccs"].shape[1], + "desc": feat["desc"].shape[1], + "comp": 5, "phys": 12, "help": 4, "exp": 32, + } + + +def _batch(B: int = 2) -> Tuple[List[str], Dict[str, torch.Tensor]]: + smiles = ["CCO", "CCN"][:B] + tabular = { + "comp": torch.randn(B, 5), + "phys": torch.randn(B, 12), + "help": torch.randn(B, 4), + "exp": torch.randn(B, 32), + } + return smiles, tabular + + +def _check_outputs(outputs: Dict[str, torch.Tensor], B: int) -> None: + for task, dim in EXPECTED.items(): + assert outputs[task].shape == (B, dim), f"{task}: {outputs[task].shape} != ({B}, {dim})" + + +def test_forward_base() -> None: + model = LNPModelWithoutMPNN(d_model=64, input_dims=_input_dims()).eval() + smiles, tabular = _batch() + _check_outputs(model(smiles, tabular), B=2) + + +def test_forward_moe() -> None: + model = LNPModelWithoutMPNN(d_model=64, input_dims=_input_dims(), use_moe=True).eval() + smiles, tabular = _batch() + _check_outputs(model(smiles, tabular), B=2) + assert model.get_last_moe_extras() is not None, "use_moe=True 应产生 MoE extras" + + +def test_forward_from_projected() -> None: + model = LNPModelWithoutMPNN(d_model=64, input_dims=_input_dims()).eval() + smiles, tabular = _batch() + stacked = model._encode_and_project(smiles, tabular) + out = model.forward_from_projected(stacked, task="biodist") + assert out.shape == (2, 7), f"biodist {out.shape} != (2, 7)" + + +def test_split_idx_no_mpnn() -> None: + model = LNPModelWithoutMPNN(d_model=64, input_dims=_input_dims()) + assert model.split_idx == 3, "无 MPNN 时化学 token 应为 3" + + +def test_forward_llm() -> None: + """需要 MolT5；缺失则跳过。""" + try: + model = LNPModelWithoutMPNN( + d_model=64, input_dims=_input_dims(), + use_moe=True, use_llm=True, llm_model_path=MOLT5_PATH, + ).eval() + except Exception as e: + print(f"[SKIP] test_forward_llm（MolT5 不可用）：{e}") + return + smiles, tabular = _batch() + _check_outputs(model(smiles, tabular), B=2) + # 冻结的 MolT5 encoder 不应进入 backbone state dict + bb = model.get_backbone_state_dict() + assert not any(k.startswith("llm_prompt.encoder.") for k in bb), "backbone 不应含冻结 encoder 权重" + + +# ────────────────────────────────────────────────────────────────────── +# Runner +# ────────────────────────────────────────────────────────────────────── + +def _collect_tests() -> List[Tuple[str, Callable[[], None]]]: + return [ + (name, fn) for name, fn in globals().items() + if name.startswith("test_") and callable(fn) + ] + + +def main() -> int: + tests = _collect_tests() + pass_count = 0 + fail: List[Tuple[str, str]] = [] + for name, fn in tests: + try: + fn() + except Exception: + fail.append((name, traceback.format_exc())) + print(f"[FAIL] {name}") + else: + pass_count += 1 + print(f"[ OK ] {name}") + print(f"\n{'='*60}") + print(f"Passed: {pass_count}/{len(tests)}") + if fail: + print(f"\n{'='*60}\nFailures:") + for name, tb in fail: + print(f"\n--- {name} ---\n{tb}") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/tests/test_moe.py b/tests/test_moe.py new file mode 100644 index 0000000..c00db47 --- /dev/null +++ b/tests/test_moe.py @@ -0,0 +1,173 @@ +"""MoE 模块自检脚本 + +约定: + - 每个 test_xxx 函数返回 None；失败时直接抛 AssertionError 或 RuntimeError。 + - main() 收集所有 test_xxx 函数依次执行，统计 pass/fail 并以非零状态退出。 +""" + +from __future__ import annotations + +import sys +import traceback +from typing import Callable, List, Tuple + +import torch + +from lnp_ml.modeling.layers.moe import MoEAttentionPool, MoEBlock, MoERouter + + +# ────────────────────────────────────────────────────────────────────── +# Helpers +# ────────────────────────────────────────────────────────────────────── + +def _expect_value_error(fn: Callable[[], object], match: str = "") -> None: + """断言调用 fn 会抛 ValueError，且消息包含 match（若给定）。""" + try: + fn() + except ValueError as e: + if match and match not in str(e): + raise AssertionError( + f"ValueError 抛出了，但消息中未包含 {match!r}: {e}" + ) + return + raise AssertionError(f"期望抛出 ValueError，但 {fn} 正常返回") + + +# ────────────────────────────────────────────────────────────────────── +# Tests +# ────────────────────────────────────────────────────────────────────── + +def test_moe_block_shape() -> None: + B, T_chem, T_tab, d, K = 2, 4, 4, 256, 4 + block = MoEBlock(d_model=d, n_chem_tokens=T_chem, n_experts=K, top_k=2) + chem = torch.randn(B, T_chem, d) + tab = torch.randn(B, T_tab, d) + + F_moe, extras = block(chem, tab) + + assert F_moe.shape == (B, d), f"F_moe shape {F_moe.shape} != ({B}, {d})" + assert extras["lb_loss"].dim() == 0 + assert extras["lb_loss"].requires_grad + assert extras["gates"].shape == (B, K) + assert extras["probs"].shape == (B, K) + + +def test_moe_block_nchem3() -> None: + B, T_chem, T_tab, d, K = 2, 3, 4, 256, 4 + block = MoEBlock(d_model=d, n_chem_tokens=T_chem, n_experts=K, top_k=2) + chem = torch.randn(B, T_chem, d) + tab = torch.randn(B, T_tab, d) + + F_moe, extras = block(chem, tab) + + assert F_moe.shape == (B, d), f"F_moe shape {F_moe.shape} != ({B}, {d})" + assert block.experts[0].net[0].in_features == T_chem * d, "expert 输入维未随 n_chem 自适应" + assert extras["gates"].shape == (B, K) + + +def test_moe_gates_topk_normalization() -> None: + block = MoEBlock(d_model=64, n_chem_tokens=4, n_experts=4, top_k=2) + chem = torch.randn(8, 4, 64) + tab = torch.randn(8, 4, 64) + _, extras = block(chem, tab) + gates = extras["gates"] + + assert torch.allclose(gates.sum(dim=-1), torch.ones(8), atol=1e-5), \ + f"gates 每行未归一为 1: {gates.sum(dim=-1)}" + nonzero_per_row = (gates > 0).sum(dim=-1).max().item() + assert nonzero_per_row <= 2, f"top_k=2 但某行有 {nonzero_per_row} 个非零" + + +def test_moe_load_balancing_loss_uniform() -> None: + """随机初始化 + 大 batch，lb_loss 应接近 Switch 风格的 uniform 极限。 + + 注意: + Switch 风格 lb_loss = K * Σ f_i * p_i。 + - Σ f_i = top_k （top-k mask 的列和约束） + - Σ p_i = 1 + - 完全 uniform 极限 ⇒ lb_loss = top_k + 随机初始化下 f 与 p 因 top-k 选择天然正相关， + 实际值会略高于 top_k，但远低于 K（K 是塌缩极限）。 + """ + torch.manual_seed(42) + block = MoEBlock(d_model=64, n_chem_tokens=4, n_experts=4, top_k=2) + chem = torch.randn(64, 4, 64) + tab = torch.randn(64, 4, 64) + _, extras = block(chem, tab) + lb = extras["lb_loss"].item() + + expected = block.top_k + upper = (block.top_k + block.n_experts) / 2 # uniform 与塌缩的中点 + assert expected * 0.8 <= lb <= upper, ( + f"lb_loss={lb:.3f} 偏离理想 uniform 值 {expected} 过远" + f"（健康区间 ~[{expected*0.8:.2f}, {upper:.2f}]）" + ) + +def test_moe_invalid_top_k() -> None: + _expect_value_error( + lambda: MoEBlock(d_model=64, n_chem_tokens=4, n_experts=4, top_k=5) + ) + + +def test_moe_chem_token_mismatch() -> None: + block = MoEBlock(d_model=64, n_chem_tokens=4, n_experts=4, top_k=2) + chem = torch.randn(2, 3, 64) + tab = torch.randn(2, 4, 64) + _expect_value_error(lambda: block(chem, tab), match="chem token") + + +def test_moe_router_eval_no_jitter() -> None: + """评估态下 jitter 不生效：相同输入应得相同 gates。""" + router = MoERouter(d_model=32, n_experts=4, top_k=2, jitter_noise=0.5) + router.eval() + q = torch.randn(4, 32) + g1, _, _ = router(q) + g2, _, _ = router(q) + assert torch.allclose(g1, g2), "eval 模式下相同输入产生了不同 gates" + + +def test_moe_attention_pool_shape() -> None: + pool = MoEAttentionPool(d_model=128) + x = torch.randn(3, 5, 128) + out = pool(x) + assert out.shape == (3, 128), f"pool out shape {out.shape} != (3, 128)" + + +# ────────────────────────────────────────────────────────────────────── +# Runner +# ────────────────────────────────────────────────────────────────────── + +def _collect_tests() -> List[Tuple[str, Callable[[], None]]]: + return [ + (name, fn) for name, fn in globals().items() + if name.startswith("test_") and callable(fn) + ] + + +def main() -> int: + tests = _collect_tests() + pass_count = 0 + fail: List[Tuple[str, str]] = [] + + for name, fn in tests: + try: + fn() + except Exception: + fail.append((name, traceback.format_exc())) + print(f"[FAIL] {name}") + else: + pass_count += 1 + print(f"[ OK ] {name}") + + print(f"\n{'='*60}") + print(f"Passed: {pass_count}/{len(tests)}") + if fail: + print(f"\n{'='*60}\nFailures:") + for name, tb in fail: + print(f"\n--- {name} ---\n{tb}") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/tests/test_set_transformer.py b/tests/test_set_transformer.py new file mode 100644 index 0000000..60ff535 --- /dev/null +++ b/tests/test_set_transformer.py @@ -0,0 +1,99 @@ +"""Set Transformer 模块自检脚本 + +约定: + - 每个 test_xxx 函数返回 None；失败时直接抛 AssertionError 或 RuntimeError。 + - main() 收集所有 test_xxx 函数依次执行，统计 pass/fail 并以非零状态退出。 +""" + +from __future__ import annotations + +import sys +import traceback +from typing import Callable, List, Tuple + +import torch + +from lnp_ml.modeling.layers.set_transformer import SetTransformer + + +# ────────────────────────────────────────────────────────────────────── +# Tests +# ────────────────────────────────────────────────────────────────────── + +def test_shape_preserved() -> None: + st = SetTransformer(d_model=256, num_heads=8, n_layers=4) + x = torch.randn(2, 4, 256) + out = st(x) + assert out.shape == (2, 4, 256), f"out shape {out.shape} != (2, 4, 256)" + + +def test_variable_set_size() -> None: + """n_chem=3（无 MPNN）和 4（有 MPNN）都要支持。""" + st = SetTransformer(d_model=64, num_heads=8, n_layers=2) + for n in (3, 4): + x = torch.randn(5, n, 64) + assert st(x).shape == (5, n, 64), f"n={n} 形状不符" + + +def test_permutation_equivariance() -> None: + """无位置编码，输出应随输入 token 置换而同步置换。""" + torch.manual_seed(0) + st = SetTransformer(d_model=64, num_heads=8, n_layers=3).eval() + x = torch.randn(2, 4, 64) + perm = torch.tensor([2, 0, 3, 1]) + out = st(x) + out_perm = st(x[:, perm, :]) + assert torch.allclose(out[:, perm, :], out_perm, atol=1e-4), "未满足置换等变性" + + +def test_grad_flows() -> None: + st = SetTransformer(d_model=32, num_heads=4, n_layers=2) + x = torch.randn(3, 4, 32, requires_grad=True) + st(x).sum().backward() + assert x.grad is not None and torch.isfinite(x.grad).all(), "梯度异常" + + +def test_isab_shape() -> None: + st = SetTransformer(d_model=64, num_heads=8, n_layers=2, block="isab", num_inducing=8) + x = torch.randn(2, 4, 64) + assert st(x).shape == (2, 4, 64), "ISAB 形状不符" + + +# ────────────────────────────────────────────────────────────────────── +# Runner +# ────────────────────────────────────────────────────────────────────── + +def _collect_tests() -> List[Tuple[str, Callable[[], None]]]: + return [ + (name, fn) for name, fn in globals().items() + if name.startswith("test_") and callable(fn) + ] + + +def main() -> int: + tests = _collect_tests() + pass_count = 0 + fail: List[Tuple[str, str]] = [] + + for name, fn in tests: + try: + fn() + except Exception: + fail.append((name, traceback.format_exc())) + print(f"[FAIL] {name}") + else: + pass_count += 1 + print(f"[ OK ] {name}") + + print(f"\n{'='*60}") + print(f"Passed: {pass_count}/{len(tests)}") + if fail: + print(f"\n{'='*60}\nFailures:") + for name, tb in fail: + print(f"\n--- {name} ---\n{tb}") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file