From 7eff2b3d02ff2d0026f4a4daca6084a99fffbaa0 Mon Sep 17 00:00:00 2001 From: DicongLi <2024379585@qq.com> Date: Thu, 23 Jul 2026 20:18:53 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=20MolFormer/GraphMVP?= =?UTF-8?q?/GROVER=20=E7=A6=BB=E7=BA=BF=20embedding=20=E4=BD=9C=E4=B8=BA?= =?UTF-8?q?=E5=8C=96=E5=AD=A6=20token?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 三个预训练分子编码器作为额外的 chemical token 接入（三选一，互斥）： - MolFormer-XL 768d（SMILES 序列，11亿分子预训练） - GraphMVP 300d（2D 图自监督，ICLR'22） - GROVER 3200d（图 Transformer，NeurIPS'20） 实现：embedding 离线编码存 npy，模型内按 {smiles: vector} 查表注入， 不改动前向逻辑，默认全关、不影响既有实验。 - models.py: 新增三个 CHEM_KEYS_WITH_*、统一的 _load_offline_emb 查表、 proj_input_dims 按开关裁剪、forward 注入；子类同步转发参数 - nested_cv_optuna.py / pretrain.py: 新增 --{molformer,graphmvp,grover}-emb/-csv - scripts_run/encode_embeddings/: 三个离线编码脚本 + README（含权重来源与 新旧环境兼容补丁说明） - results/pretrained_encoders/: 五组 15 折结果 summary (trials15/inner3/repeats3/seed42, 均带 external 预训练) 注：npy embedding 与预训练权重未入库（可由编码脚本复现） --- lnp_ml/modeling/models.py | 65 +- lnp_ml/modeling/nested_cv_optuna.py | 13 + lnp_ml/modeling/pretrain.py | 1173 +++++++++-------- .../baseline_pretrain_s42_summary.json | 932 +++++++++++++ .../graphmvp_pretrain_s42_summary.json | 932 +++++++++++++ .../grover_pretrain_s42_summary.json | 932 +++++++++++++ .../moe_pretrain_s42_summary.json | 977 ++++++++++++++ .../molformer_pretrain_s42_summary.json | 932 +++++++++++++ scripts_run/encode_embeddings/README.md | 43 + .../encode_embeddings/encode_graphmvp.py | 122 ++ .../encode_embeddings/encode_grover.py | 57 + .../encode_embeddings/encode_molformer.py | 51 + 12 files changed, 5654 insertions(+), 575 deletions(-) create mode 100644 results/pretrained_encoders/baseline_pretrain_s42_summary.json create mode 100644 results/pretrained_encoders/graphmvp_pretrain_s42_summary.json create mode 100644 results/pretrained_encoders/grover_pretrain_s42_summary.json create mode 100644 results/pretrained_encoders/moe_pretrain_s42_summary.json create mode 100644 results/pretrained_encoders/molformer_pretrain_s42_summary.json create mode 100644 scripts_run/encode_embeddings/README.md create mode 100644 scripts_run/encode_embeddings/encode_graphmvp.py create mode 100644 scripts_run/encode_embeddings/encode_grover.py create mode 100644 scripts_run/encode_embeddings/encode_molformer.py diff --git a/lnp_ml/modeling/models.py b/lnp_ml/modeling/models.py index 1428075..2bdd956 100644 --- a/lnp_ml/modeling/models.py +++ b/lnp_ml/modeling/models.py @@ -32,6 +32,9 @@ _DESC_DIM = _infer_desc_dim() # 本机 = 210 DEFAULT_INPUT_DIMS = { # Channel A: 化学特征 "mpnn": 600, # D-MPNN embedding + "molformer": 768, # MolFormer-XL 离线 embedding + "graphmvp": 300, # GraphMVP GIN 离线 embedding + "grover": 3200, # GROVER dualtrans fingerprint (atom+bond) "morgan": 1024, # Morgan fingerprint "maccs": 167, # MACCS keys "desc": _DESC_DIM, # RDKit descriptors（随版本动态） @@ -45,6 +48,9 @@ DEFAULT_INPUT_DIMS = { # 化学 / 配方 token 的键顺序 CHEM_KEYS_WITH_MPNN = ["mpnn", "morgan", "maccs", "desc"] CHEM_KEYS_NO_MPNN = ["morgan", "maccs", "desc"] +CHEM_KEYS_WITH_MOLFORMER = ["molformer", "morgan", "maccs", "desc"] +CHEM_KEYS_WITH_GRAPHMVP = ["graphmvp", "morgan", "maccs", "desc"] +CHEM_KEYS_WITH_GROVER = ["grover", "morgan", "maccs", "desc"] TAB_KEYS = ["comp", "phys", "help", "exp"] # backbone 权重前缀（用于预训练加载与导出） @@ -115,6 +121,13 @@ class LNPModel(nn.Module): rag_top_k: int = 4, use_retrieval: bool = False, retr_feature_dim: int = 0, + # ===== 离线预训练分子 embedding（三选一）===== + molformer_emb_path: Optional[str] = None, + molformer_smiles_csv: Optional[str] = None, + graphmvp_emb_path: Optional[str] = None, + graphmvp_smiles_csv: Optional[str] = None, + grover_emb_path: Optional[str] = None, + grover_smiles_csv: Optional[str] = None, ) -> None: super().__init__() @@ -122,6 +135,22 @@ class LNPModel(nn.Module): self.d_model = d_model self.use_mpnn = mpnn_checkpoint is not None or mpnn_ensemble_paths is not None + # 离线预训练 embedding：{smiles: vector} 查表 + def _load_offline_emb(emb_path, csv_path): + if emb_path is None: + return False, {} + import numpy as _np, pandas as _pd + _E = _np.load(emb_path) + _sm = _pd.read_csv(csv_path, low_memory=False)["smiles"].astype(str).tolist() + assert len(_sm) == _E.shape[0], f"SMILES({len(_sm)}) vs emb({_E.shape[0]}) 不匹配" + return True, {sm: _E[i] for i, sm in enumerate(_sm)} + + self.use_molformer, self._molformer_lut = _load_offline_emb(molformer_emb_path, molformer_smiles_csv) + self.use_graphmvp, self._graphmvp_lut = _load_offline_emb(graphmvp_emb_path, graphmvp_smiles_csv) + self.use_grover, self._grover_lut = _load_offline_emb(grover_emb_path, grover_smiles_csv) + assert sum([self.use_molformer, self.use_graphmvp, self.use_grover]) <= 1, \ + "molformer/graphmvp/grover 只能启用其中一个" + # ============ Encoders ============ self.rdkit_encoder = CachedRDKitEncoder() if self.use_mpnn: @@ -137,6 +166,9 @@ class LNPModel(nn.Module): proj_input_dims = {k: v for k, v in self.input_dims.items()} if not self.use_mpnn: proj_input_dims.pop("mpnn", None) + for _k, _flag in (("molformer", "use_molformer"), ("graphmvp", "use_graphmvp"), ("grover", "use_grover")): + if not getattr(self, _flag, False): + proj_input_dims.pop(_k, None) self.token_projector = TokenProjector( input_dims=proj_input_dims, d_model=d_model, @@ -144,7 +176,16 @@ class LNPModel(nn.Module): ) # token 顺序与化学侧 token 数 - self.chem_keys = CHEM_KEYS_WITH_MPNN if self.use_mpnn else CHEM_KEYS_NO_MPNN + if self.use_mpnn: + self.chem_keys = CHEM_KEYS_WITH_MPNN + elif self.use_molformer: + self.chem_keys = CHEM_KEYS_WITH_MOLFORMER + elif self.use_graphmvp: + self.chem_keys = CHEM_KEYS_WITH_GRAPHMVP + elif self.use_grover: + self.chem_keys = CHEM_KEYS_WITH_GROVER + else: + self.chem_keys = CHEM_KEYS_NO_MPNN self.tab_keys = TAB_KEYS self.token_order = self.chem_keys + self.tab_keys self.split_idx = len(self.chem_keys) @@ -233,6 +274,16 @@ class LNPModel(nn.Module): if self.use_mpnn: mpnn_features = self.mpnn_encoder(smiles) all_features["mpnn"] = mpnn_features["mpnn"].to(device) + import numpy as _np + for _key, _flag, _lut in ( + ("molformer", "use_molformer", "_molformer_lut"), + ("graphmvp", "use_graphmvp", "_graphmvp_lut"), + ("grover", "use_grover", "_grover_lut"), + ): + if getattr(self, _flag, False): + _lookup = getattr(self, _lut) + _v = _np.stack([_lookup[sm] for sm in smiles]) + all_features[_key] = torch.from_numpy(_v).float().to(device) all_features["morgan"] = rdkit_features["morgan"].to(device) all_features["maccs"] = rdkit_features["maccs"].to(device) all_features["desc"] = rdkit_features["desc"].to(device) @@ -464,6 +515,12 @@ class LNPModelWithoutMPNN(LNPModel): rag_top_k: int = 4, use_retrieval: bool = False, retr_feature_dim: int = 0, + molformer_emb_path: Optional[str] = None, + molformer_smiles_csv: Optional[str] = None, + graphmvp_emb_path: Optional[str] = None, + graphmvp_smiles_csv: Optional[str] = None, + grover_emb_path: Optional[str] = None, + grover_smiles_csv: Optional[str] = None, ) -> None: dims = input_dims or DEFAULT_INPUT_DIMS.copy() dims.pop("mpnn", None) @@ -498,4 +555,10 @@ class LNPModelWithoutMPNN(LNPModel): llm_lora_r=llm_lora_r, llm_lora_alpha=llm_lora_alpha, llm_lora_dropout=llm_lora_dropout, + molformer_emb_path=molformer_emb_path, + molformer_smiles_csv=molformer_smiles_csv, + graphmvp_emb_path=graphmvp_emb_path, + graphmvp_smiles_csv=graphmvp_smiles_csv, + grover_emb_path=grover_emb_path, + grover_smiles_csv=grover_smiles_csv, ) \ No newline at end of file diff --git a/lnp_ml/modeling/nested_cv_optuna.py b/lnp_ml/modeling/nested_cv_optuna.py index 7c34e63..bc41f86 100644 --- a/lnp_ml/modeling/nested_cv_optuna.py +++ b/lnp_ml/modeling/nested_cv_optuna.py @@ -932,6 +932,13 @@ def main( set_transformer_block: str = "sab", # 回归旁路（消融开关） reg_bypass: str = "on", + # 离线预训练分子 embedding（三选一） + molformer_emb: Optional[str] = None, + molformer_csv: Optional[str] = None, + graphmvp_emb: Optional[str] = None, + graphmvp_csv: Optional[str] = None, + grover_emb: Optional[str] = None, + grover_csv: Optional[str] = None, # LLM（消融开关） use_llm: bool = False, use_rag: bool = False, @@ -970,6 +977,12 @@ def main( llm_kwargs = dict( reg_bypass=reg_bypass, + molformer_emb_path=molformer_emb, + molformer_smiles_csv=molformer_csv, + graphmvp_emb_path=graphmvp_emb, + graphmvp_smiles_csv=graphmvp_csv, + grover_emb_path=grover_emb, + grover_smiles_csv=grover_csv, use_rag=use_rag, rag_top_k=rag_top_k, use_llm=use_llm, diff --git a/lnp_ml/modeling/pretrain.py b/lnp_ml/modeling/pretrain.py index 54a39ea..ea552f1 100644 --- a/lnp_ml/modeling/pretrain.py +++ b/lnp_ml/modeling/pretrain.py @@ -1,574 +1,599 @@ -"""预训练脚本：在外部 LiON 数据上预训练 backbone + delivery head""" - -import json -from pathlib import Path -from typing import Dict, List, Optional - -import pandas as pd -import torch -import torch.nn as nn -from torch.utils.data import DataLoader -from loguru import logger -from tqdm import tqdm -import typer - -from lnp_ml.config import MODELS_DIR, PROCESSED_DATA_DIR -from lnp_ml.dataset import ExternalDeliveryDataset, collate_fn -from lnp_ml.modeling.visualization import plot_loss_curves - -# MPNN ensemble 默认路径 -DEFAULT_MPNN_ENSEMBLE_DIR = MODELS_DIR / "mpnn" / "all_amine_split_for_LiON" - - -def find_mpnn_ensemble_paths(base_dir: Path = DEFAULT_MPNN_ENSEMBLE_DIR) -> List[str]: - """ - 自动查找 MPNN ensemble 的 model.pt 文件。 - - 在 base_dir 下查找所有 cv_*/fold_*/model_*/model.pt 文件。 - """ - model_paths = sorted(base_dir.glob("cv_*/fold_*/model_*/model.pt")) - if not model_paths: - raise FileNotFoundError(f"No model.pt files found in {base_dir}") - return [str(p) for p in model_paths] -from lnp_ml.modeling.models import LNPModel, LNPModelWithoutMPNN -from lnp_ml.modeling.layers.llm_prompt import DEFAULT_MOLT5_PATH -from lnp_ml.utils.seed import set_global_seed - - -app = typer.Typer() - - -class EarlyStopping: - """早停机制""" - - def __init__(self, patience: int = 10, min_delta: float = 0.0): - self.patience = patience - self.min_delta = min_delta - self.counter = 0 - self.best_loss = float("inf") - - def __call__(self, val_loss: float) -> bool: - if val_loss < self.best_loss - self.min_delta: - self.best_loss = val_loss - self.counter = 0 - return False - self.counter += 1 - return self.counter >= self.patience - - -def warmup_cache(model: nn.Module, smiles_list: List[str], batch_size: int = 256) -> None: - """ - 预热 RDKit 特征缓存，避免训练时计算阻塞。 - """ - unique_smiles = list(set(smiles_list)) - logger.info(f"Warming up RDKit cache for {len(unique_smiles)} unique SMILES...") - - for i in tqdm(range(0, len(unique_smiles), batch_size), desc="Cache warmup"): - batch = unique_smiles[i:i + batch_size] - model.rdkit_encoder(batch) - - logger.success(f"Cache warmup complete. Cached {len(model.rdkit_encoder._cache)} SMILES.") - - -def train_epoch_delivery( - model: nn.Module, - loader: DataLoader, - optimizer: torch.optim.Optimizer, - device: torch.device, - epoch: int = 0, -) -> Dict[str, float]: - """ - 单个 epoch 的预训练（仅 delivery 任务）。 - """ - model.train() - total_loss = 0.0 - n_samples = 0 - - pbar = tqdm(loader, desc=f"Epoch {epoch+1} [Train]", leave=False) - for batch in pbar: - smiles = batch["smiles"] - tabular = {k: v.to(device) for k, v in batch["tabular"].items()} - targets = batch["targets"]["delivery"].to(device) - mask = batch["mask"]["delivery"].to(device) - - optimizer.zero_grad() - - # Forward: 只预测 delivery - pred = model.forward_delivery(smiles, tabular) # [B, 1] - pred = pred.squeeze(-1) # [B] - - # 计算损失（仅对有效样本） - if mask.any(): - loss = nn.functional.mse_loss(pred[mask], targets[mask]) - loss.backward() - optimizer.step() - total_loss += loss.item() * mask.sum().item() - n_samples += mask.sum().item() - - pbar.set_postfix({"loss": total_loss / max(n_samples, 1)}) - - avg_loss = total_loss / max(n_samples, 1) - return {"loss": avg_loss, "n_samples": n_samples} - - -@torch.no_grad() -def validate_delivery( - model: nn.Module, - loader: DataLoader, - device: torch.device, -) -> Dict[str, float]: - """ - 验证（仅 delivery 任务）。 - """ - model.eval() - total_loss = 0.0 - n_samples = 0 - - for batch in loader: - smiles = batch["smiles"] - tabular = {k: v.to(device) for k, v in batch["tabular"].items()} - targets = batch["targets"]["delivery"].to(device) - mask = batch["mask"]["delivery"].to(device) - - pred = model.forward_delivery(smiles, tabular).squeeze(-1) - - if mask.any(): - loss = nn.functional.mse_loss(pred[mask], targets[mask]) - total_loss += loss.item() * mask.sum().item() - n_samples += mask.sum().item() - - avg_loss = total_loss / max(n_samples, 1) - return {"loss": avg_loss, "n_samples": n_samples} - - -def pretrain( - train_loader: DataLoader, - val_loader: DataLoader, - model: nn.Module, - device: torch.device, - lr: float = 1e-4, - weight_decay: float = 1e-5, - epochs: int = 50, - patience: int = 10, -) -> dict: - """ - 预训练循环。 - - Returns: - 训练历史和最佳验证损失 - """ - model = model.to(device) - trainable_params = [p for p in model.parameters() if p.requires_grad] - optimizer = torch.optim.AdamW(trainable_params, lr=lr, weight_decay=weight_decay) - scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( - optimizer, mode="min", factor=0.5, patience=5 - ) - early_stopping = EarlyStopping(patience=patience) - - history = {"train": [], "val": []} - best_val_loss = float("inf") - best_state = None - - for epoch in range(epochs): - # Train - train_metrics = train_epoch_delivery(model, train_loader, optimizer, device, epoch) - - # Validate - val_metrics = validate_delivery(model, val_loader, device) - - # Log - logger.info( - f"Epoch {epoch + 1}/{epochs} | " - f"Train Loss: {train_metrics['loss']:.4f} | " - f"Val Loss: {val_metrics['loss']:.4f}" - ) - - history["train"].append(train_metrics) - history["val"].append(val_metrics) - - # Learning rate scheduling - scheduler.step(val_metrics["loss"]) - - # Save best model - if val_metrics["loss"] < best_val_loss: - best_val_loss = val_metrics["loss"] - best_state = {k: v.cpu().clone() for k, v in model.state_dict().items()} - logger.info(f" -> New best model (val_loss={best_val_loss:.4f})") - - # Early stopping - if early_stopping(val_metrics["loss"]): - logger.info(f"Early stopping at epoch {epoch + 1}") - break - - # Restore best model - if best_state is not None: - model.load_state_dict(best_state) - - return { - "history": history, - "best_val_loss": best_val_loss, - } - - -@app.command() -def main( - # 数据路径（已处理的 parquet 文件） - train_path: Path = PROCESSED_DATA_DIR / "train_pretrain.parquet", - val_path: Path = PROCESSED_DATA_DIR / "val_pretrain.parquet", - output_dir: Path = MODELS_DIR, - # 模型参数 - d_model: int = 256, - num_heads: int = 8, - n_attn_layers: int = 4, - set_transformer_block: str = "sab", # "sab" | "isab" - fusion_strategy: str = "attention", - head_hidden_dim: int = 128, - dropout: float = 0.1, - # MoE 参数（消融开关） - use_moe: bool = False, - moe_n_experts: int = 4, - moe_top_k: int = 2, - moe_expert_hidden_mult: int = 2, - moe_jitter_noise: float = 0.0, - # LLM 参数（消融开关） - use_llm: bool = False, - llm_model_path: str = DEFAULT_MOLT5_PATH, - llm_freeze: bool = True, - llm_use_lora: bool = False, - llm_lora_r: int = 8, - llm_lora_alpha: int = 16, - llm_lora_dropout: float = 0.05, - # MPNN 参数（可选） - use_mpnn: bool = False, # 启用 MPNN，自动从默认路径加载 ensemble - mpnn_checkpoint: Optional[str] = None, - mpnn_ensemble_paths: Optional[str] = None, - mpnn_device: str = "cpu", - # 训练参数 - batch_size: int = 64, - lr: float = 1e-4, - weight_decay: float = 1e-5, - epochs: int = 50, - patience: int = 10, - # 随机种子 - seed: int = 42, - # 设备 - device: str = "cuda" if torch.cuda.is_available() else "cpu", -): - """ - 在外部 LiON 数据上预训练 backbone + delivery head。 - - 需要先运行 `make data_pretrain` 生成 parquet 文件。 - - 使用 --use-mpnn 启用 MPNN encoder（自动从 models/mpnn/all_amine_split_for_LiON 加载）。 - - 产出: - - models/pretrain_delivery.pt: 包含 backbone + delivery head 权重 - - models/pretrain_history.json: 训练历史 - """ - set_global_seed(seed) - logger.info(f"Using device: {device} | seed: {seed}") - device_obj = torch.device(device) - - # 加载已处理的 parquet 文件 - logger.info(f"Loading train data from {train_path}") - train_df = pd.read_parquet(train_path) - train_dataset = ExternalDeliveryDataset(train_df) - - logger.info(f"Loading val data from {val_path}") - val_df = pd.read_parquet(val_path) - val_dataset = ExternalDeliveryDataset(val_df) - - logger.info(f"Train samples: {len(train_dataset)}, Val samples: {len(val_dataset)}") - - train_loader = DataLoader( - train_dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_fn - ) - val_loader = DataLoader( - val_dataset, batch_size=batch_size, shuffle=False, collate_fn=collate_fn - ) - - # 解析 MPNN 配置 - # 优先级：mpnn_checkpoint > mpnn_ensemble_paths > use_mpnn（自动查找） - ensemble_paths_list = None - if mpnn_ensemble_paths: - ensemble_paths_list = mpnn_ensemble_paths.split(",") - elif use_mpnn and mpnn_checkpoint is None: - # --use-mpnn 但没有指定具体路径，自动查找 - logger.info(f"Auto-detecting MPNN ensemble from {DEFAULT_MPNN_ENSEMBLE_DIR}") - ensemble_paths_list = find_mpnn_ensemble_paths() - logger.info(f"Found {len(ensemble_paths_list)} MPNN models") - - enable_mpnn = mpnn_checkpoint is not None or ensemble_paths_list is not None - - # 创建模型 - logger.info( - f"Creating model (use_mpnn={enable_mpnn}, use_moe={use_moe}, use_llm={use_llm})..." - ) - common_kwargs = dict( - d_model=d_model, - num_heads=num_heads, - n_attn_layers=n_attn_layers, - set_transformer_block=set_transformer_block, - fusion_strategy=fusion_strategy, - head_hidden_dim=head_hidden_dim, - dropout=dropout, - use_moe=use_moe, - moe_n_experts=moe_n_experts, - moe_top_k=moe_top_k, - moe_expert_hidden_mult=moe_expert_hidden_mult, - moe_jitter_noise=moe_jitter_noise, - use_llm=use_llm, - llm_model_path=llm_model_path, - llm_freeze=llm_freeze, - llm_use_lora=llm_use_lora, - llm_lora_r=llm_lora_r, - llm_lora_alpha=llm_lora_alpha, - llm_lora_dropout=llm_lora_dropout, - ) - if enable_mpnn: - model = LNPModel( - mpnn_checkpoint=mpnn_checkpoint, - mpnn_ensemble_paths=ensemble_paths_list, - mpnn_device=mpnn_device, - **common_kwargs, - ) - else: - model = LNPModelWithoutMPNN(**common_kwargs) - - n_params_total = sum(p.numel() for p in model.parameters()) - n_params_trainable = sum(p.numel() for p in model.parameters() if p.requires_grad) - logger.info(f"Model parameters: {n_params_total:,} total, {n_params_trainable:,} trainable") - - # 预热 RDKit 缓存（避免训练时阻塞） - all_smiles = train_df["smiles"].tolist() + val_df["smiles"].tolist() - warmup_cache(model, all_smiles, batch_size=256) - - # 预训练 - logger.info("Starting pretraining on external data (delivery only)...") - result = pretrain( - train_loader=train_loader, - val_loader=val_loader, - model=model, - device=device_obj, - lr=lr, - weight_decay=weight_decay, - epochs=epochs, - patience=patience, - ) - - # 保存预训练 checkpoint - output_dir.mkdir(parents=True, exist_ok=True) - checkpoint_path = output_dir / "pretrain_delivery.pt" - torch.save( - { - "model_state_dict": model.state_dict(), - "backbone_state_dict": model.get_backbone_state_dict(), - "delivery_head_state_dict": model.get_delivery_head_state_dict(), - "config": { - "d_model": d_model, - "num_heads": num_heads, - "n_attn_layers": n_attn_layers, - "set_transformer_block": set_transformer_block, - "fusion_strategy": fusion_strategy, - "head_hidden_dim": head_hidden_dim, - "dropout": dropout, - "use_mpnn": enable_mpnn, - "use_moe": use_moe, - "moe_n_experts": moe_n_experts, - "moe_top_k": moe_top_k, - "moe_expert_hidden_mult": moe_expert_hidden_mult, - "moe_jitter_noise": moe_jitter_noise, - "use_llm": use_llm, - "llm_model_path": llm_model_path, - "llm_freeze": llm_freeze, - "llm_use_lora": llm_use_lora, - "llm_lora_r": llm_lora_r, - "llm_lora_alpha": llm_lora_alpha, - "llm_lora_dropout": llm_lora_dropout, - }, - "best_val_loss": result["best_val_loss"], - }, - checkpoint_path, - ) - logger.success(f"Saved pretrain checkpoint to {checkpoint_path}") - - # 保存训练历史 - history_path = output_dir / "pretrain_history.json" - with open(history_path, "w") as f: - json.dump(result["history"], f, indent=2) - logger.success(f"Saved pretrain history to {history_path}") - - # 绘制 loss 曲线图 - loss_plot_path = output_dir / "pretrain_loss_curves.png" - plot_loss_curves( - history=result["history"], - output_path=loss_plot_path, - title="Pretrain Loss Curves (Delivery)", - ) - logger.success(f"Saved loss curves plot to {loss_plot_path}") - - logger.success( - f"Pretraining complete! Best val_loss: {result['best_val_loss']:.4f}" - ) - - -@app.command() -def test( - # 数据路径 - val_path: Path = PROCESSED_DATA_DIR / "val_pretrain.parquet", - model_path: Path = MODELS_DIR / "pretrain_delivery.pt", - output_path: Path = MODELS_DIR / "pretrain_test_results.json", - # MPNN 参数 - use_mpnn: bool = False, - mpnn_device: str = "cpu", - # 其他参数 - batch_size: int = 64, - device: str = "cuda" if torch.cuda.is_available() else "cpu", -): - """ - 评估 pretrain 模型在外部数据上的 delivery 预测性能。 - - 输出详细指标：MSE, RMSE, MAE, R² - """ - import numpy as np - from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score - - logger.info(f"Using device: {device}") - device_obj = torch.device(device) - - # 加载模型 - logger.info(f"Loading pretrain model from {model_path}") - checkpoint = torch.load(model_path, map_location=device_obj, weights_only=False) - config = checkpoint["config"] - - # 解析 MPNN 配置 - enable_mpnn = config.get("use_mpnn", False) - common_kwargs = dict( - d_model=config["d_model"], - num_heads=config["num_heads"], - n_attn_layers=config["n_attn_layers"], - set_transformer_block=config.get("set_transformer_block", "sab"), - fusion_strategy=config["fusion_strategy"], - head_hidden_dim=config["head_hidden_dim"], - dropout=config["dropout"], - use_moe=config.get("use_moe", False), - moe_n_experts=config.get("moe_n_experts", 4), - moe_top_k=config.get("moe_top_k", 2), - moe_expert_hidden_mult=config.get("moe_expert_hidden_mult", 2), - moe_jitter_noise=config.get("moe_jitter_noise", 0.0), - use_llm=config.get("use_llm", False), - llm_model_path=config.get("llm_model_path", DEFAULT_MOLT5_PATH), - llm_freeze=config.get("llm_freeze", True), - llm_use_lora=config.get("llm_use_lora", False), - llm_lora_r=config.get("llm_lora_r", 8), - llm_lora_alpha=config.get("llm_lora_alpha", 16), - llm_lora_dropout=config.get("llm_lora_dropout", 0.05), - ) - if enable_mpnn or use_mpnn: - logger.info(f"Auto-detecting MPNN ensemble from {DEFAULT_MPNN_ENSEMBLE_DIR}") - ensemble_paths = find_mpnn_ensemble_paths() - logger.info(f"Found {len(ensemble_paths)} MPNN models") - model = LNPModel( - mpnn_ensemble_paths=ensemble_paths, - mpnn_device=mpnn_device, - **common_kwargs, - ) - else: - model = LNPModelWithoutMPNN(**common_kwargs) - - model.load_state_dict(checkpoint["model_state_dict"], strict=False) - model.to(device_obj) - model.eval() - - logger.info(f"Model config: {config}") - logger.info(f"Best val_loss from training: {checkpoint.get('best_val_loss', 'N/A')}") - - # 加载数据 - logger.info(f"Loading validation data from {val_path}") - val_df = pd.read_parquet(val_path) - val_dataset = ExternalDeliveryDataset(val_df) - val_loader = DataLoader( - val_dataset, batch_size=batch_size, shuffle=False, collate_fn=collate_fn - ) - logger.info(f"Validation samples: {len(val_dataset)}") - - # 预测 - logger.info("Running predictions...") - all_preds = [] - all_targets = [] - all_masks = [] - - with torch.no_grad(): - for batch in tqdm(val_loader, desc="Predicting"): - smiles = batch["smiles"] - tabular = {k: v.to(device_obj) for k, v in batch["tabular"].items()} - targets = batch["targets"]["delivery"].numpy() - mask = batch["mask"]["delivery"].numpy() - - pred = model.forward_delivery(smiles, tabular).squeeze(-1).cpu().numpy() - - all_preds.extend(pred) - all_targets.extend(targets) - all_masks.extend(mask) - - # 转为数组 - all_preds = np.array(all_preds) - all_targets = np.array(all_targets) - all_masks = np.array(all_masks, dtype=bool) - - # 只计算有效样本 - y_pred = all_preds[all_masks] - y_true = all_targets[all_masks] - - # 计算指标 - mse = float(mean_squared_error(y_true, y_pred)) - rmse = float(np.sqrt(mse)) - mae = float(mean_absolute_error(y_true, y_pred)) - r2 = float(r2_score(y_true, y_pred)) - - # 额外统计 - correlation = float(np.corrcoef(y_true, y_pred)[0, 1]) - - results = { - "model_path": str(model_path), - "val_path": str(val_path), - "n_samples": int(all_masks.sum()), - "metrics": { - "mse": mse, - "rmse": rmse, - "mae": mae, - "r2": r2, - "correlation": correlation, - }, - "statistics": { - "y_true_mean": float(y_true.mean()), - "y_true_std": float(y_true.std()), - "y_pred_mean": float(y_pred.mean()), - "y_pred_std": float(y_pred.std()), - } - } - - # 打印结果 - logger.info("\n" + "=" * 50) - logger.info("PRETRAIN MODEL EVALUATION (Delivery)") - logger.info("=" * 50) - logger.info(f"Samples: {results['n_samples']}") - logger.info("\n[Metrics]") - logger.info(f" MSE: {mse:.4f}") - logger.info(f" RMSE: {rmse:.4f}") - logger.info(f" MAE: {mae:.4f}") - logger.info(f" R²: {r2:.4f}") - logger.info(f" Correlation: {correlation:.4f}") - logger.info("\n[Statistics]") - logger.info(f" True: mean={y_true.mean():.4f}, std={y_true.std():.4f}") - logger.info(f" Pred: mean={y_pred.mean():.4f}, std={y_pred.std():.4f}") - - # 保存结果 - with open(output_path, "w") as f: - json.dump(results, f, indent=2) - logger.success(f"\nSaved results to {output_path}") - - -if __name__ == "__main__": - app() - +"""预训练脚本：在外部 LiON 数据上预训练 backbone + delivery head""" + +import json +from pathlib import Path +from typing import Dict, List, Optional + +import pandas as pd +import torch +import torch.nn as nn +from torch.utils.data import DataLoader +from loguru import logger +from tqdm import tqdm +import typer + +from lnp_ml.config import MODELS_DIR, PROCESSED_DATA_DIR +from lnp_ml.dataset import ExternalDeliveryDataset, collate_fn +from lnp_ml.modeling.visualization import plot_loss_curves + +# MPNN ensemble 默认路径 +DEFAULT_MPNN_ENSEMBLE_DIR = MODELS_DIR / "mpnn" / "all_amine_split_for_LiON" + + +def find_mpnn_ensemble_paths(base_dir: Path = DEFAULT_MPNN_ENSEMBLE_DIR) -> List[str]: + """ + 自动查找 MPNN ensemble 的 model.pt 文件。 + + 在 base_dir 下查找所有 cv_*/fold_*/model_*/model.pt 文件。 + """ + model_paths = sorted(base_dir.glob("cv_*/fold_*/model_*/model.pt")) + if not model_paths: + raise FileNotFoundError(f"No model.pt files found in {base_dir}") + return [str(p) for p in model_paths] +from lnp_ml.modeling.models import LNPModel, LNPModelWithoutMPNN +from lnp_ml.modeling.layers.llm_prompt import DEFAULT_MOLT5_PATH +from lnp_ml.utils.seed import set_global_seed + + +app = typer.Typer() + + +class EarlyStopping: + """早停机制""" + + def __init__(self, patience: int = 10, min_delta: float = 0.0): + self.patience = patience + self.min_delta = min_delta + self.counter = 0 + self.best_loss = float("inf") + + def __call__(self, val_loss: float) -> bool: + if val_loss < self.best_loss - self.min_delta: + self.best_loss = val_loss + self.counter = 0 + return False + self.counter += 1 + return self.counter >= self.patience + + +def warmup_cache(model: nn.Module, smiles_list: List[str], batch_size: int = 256) -> None: + """ + 预热 RDKit 特征缓存，避免训练时计算阻塞。 + """ + unique_smiles = list(set(smiles_list)) + logger.info(f"Warming up RDKit cache for {len(unique_smiles)} unique SMILES...") + + for i in tqdm(range(0, len(unique_smiles), batch_size), desc="Cache warmup"): + batch = unique_smiles[i:i + batch_size] + model.rdkit_encoder(batch) + + logger.success(f"Cache warmup complete. Cached {len(model.rdkit_encoder._cache)} SMILES.") + + +def train_epoch_delivery( + model: nn.Module, + loader: DataLoader, + optimizer: torch.optim.Optimizer, + device: torch.device, + epoch: int = 0, +) -> Dict[str, float]: + """ + 单个 epoch 的预训练（仅 delivery 任务）。 + """ + model.train() + total_loss = 0.0 + n_samples = 0 + + pbar = tqdm(loader, desc=f"Epoch {epoch+1} [Train]", leave=False) + for batch in pbar: + smiles = batch["smiles"] + tabular = {k: v.to(device) for k, v in batch["tabular"].items()} + targets = batch["targets"]["delivery"].to(device) + mask = batch["mask"]["delivery"].to(device) + + optimizer.zero_grad() + + # Forward: 只预测 delivery + pred = model.forward_delivery(smiles, tabular) # [B, 1] + pred = pred.squeeze(-1) # [B] + + # 计算损失（仅对有效样本） + if mask.any(): + loss = nn.functional.mse_loss(pred[mask], targets[mask]) + loss.backward() + optimizer.step() + total_loss += loss.item() * mask.sum().item() + n_samples += mask.sum().item() + + pbar.set_postfix({"loss": total_loss / max(n_samples, 1)}) + + avg_loss = total_loss / max(n_samples, 1) + return {"loss": avg_loss, "n_samples": n_samples} + + +@torch.no_grad() +def validate_delivery( + model: nn.Module, + loader: DataLoader, + device: torch.device, +) -> Dict[str, float]: + """ + 验证（仅 delivery 任务）。 + """ + model.eval() + total_loss = 0.0 + n_samples = 0 + + for batch in loader: + smiles = batch["smiles"] + tabular = {k: v.to(device) for k, v in batch["tabular"].items()} + targets = batch["targets"]["delivery"].to(device) + mask = batch["mask"]["delivery"].to(device) + + pred = model.forward_delivery(smiles, tabular).squeeze(-1) + + if mask.any(): + loss = nn.functional.mse_loss(pred[mask], targets[mask]) + total_loss += loss.item() * mask.sum().item() + n_samples += mask.sum().item() + + avg_loss = total_loss / max(n_samples, 1) + return {"loss": avg_loss, "n_samples": n_samples} + + +def pretrain( + train_loader: DataLoader, + val_loader: DataLoader, + model: nn.Module, + device: torch.device, + lr: float = 1e-4, + weight_decay: float = 1e-5, + epochs: int = 50, + patience: int = 10, +) -> dict: + """ + 预训练循环。 + + Returns: + 训练历史和最佳验证损失 + """ + model = model.to(device) + trainable_params = [p for p in model.parameters() if p.requires_grad] + optimizer = torch.optim.AdamW(trainable_params, lr=lr, weight_decay=weight_decay) + scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( + optimizer, mode="min", factor=0.5, patience=5 + ) + early_stopping = EarlyStopping(patience=patience) + + history = {"train": [], "val": []} + best_val_loss = float("inf") + best_state = None + + for epoch in range(epochs): + # Train + train_metrics = train_epoch_delivery(model, train_loader, optimizer, device, epoch) + + # Validate + val_metrics = validate_delivery(model, val_loader, device) + + # Log + logger.info( + f"Epoch {epoch + 1}/{epochs} | " + f"Train Loss: {train_metrics['loss']:.4f} | " + f"Val Loss: {val_metrics['loss']:.4f}" + ) + + history["train"].append(train_metrics) + history["val"].append(val_metrics) + + # Learning rate scheduling + scheduler.step(val_metrics["loss"]) + + # Save best model + if val_metrics["loss"] < best_val_loss: + best_val_loss = val_metrics["loss"] + best_state = {k: v.cpu().clone() for k, v in model.state_dict().items()} + logger.info(f" -> New best model (val_loss={best_val_loss:.4f})") + + # Early stopping + if early_stopping(val_metrics["loss"]): + logger.info(f"Early stopping at epoch {epoch + 1}") + break + + # Restore best model + if best_state is not None: + model.load_state_dict(best_state) + + return { + "history": history, + "best_val_loss": best_val_loss, + } + + +@app.command() +def main( + # 数据路径（已处理的 parquet 文件） + train_path: Path = PROCESSED_DATA_DIR / "train_pretrain.parquet", + val_path: Path = PROCESSED_DATA_DIR / "val_pretrain.parquet", + output_dir: Path = MODELS_DIR, + # 模型参数 + d_model: int = 256, + num_heads: int = 8, + n_attn_layers: int = 4, + set_transformer_block: str = "sab", # "sab" | "isab" + fusion_strategy: str = "attention", + head_hidden_dim: int = 128, + dropout: float = 0.1, + # 离线预训练分子 embedding（三选一） + molformer_emb: Optional[str] = None, + molformer_csv: Optional[str] = None, + graphmvp_emb: Optional[str] = None, + graphmvp_csv: Optional[str] = None, + grover_emb: Optional[str] = None, + grover_csv: Optional[str] = None, + # MoE 参数（消融开关） + use_moe: bool = False, + moe_n_experts: int = 4, + moe_top_k: int = 2, + moe_expert_hidden_mult: int = 2, + moe_jitter_noise: float = 0.0, + # LLM 参数（消融开关） + use_llm: bool = False, + llm_model_path: str = DEFAULT_MOLT5_PATH, + llm_freeze: bool = True, + llm_use_lora: bool = False, + llm_lora_r: int = 8, + llm_lora_alpha: int = 16, + llm_lora_dropout: float = 0.05, + # MPNN 参数（可选） + use_mpnn: bool = False, # 启用 MPNN，自动从默认路径加载 ensemble + mpnn_checkpoint: Optional[str] = None, + mpnn_ensemble_paths: Optional[str] = None, + mpnn_device: str = "cpu", + # 训练参数 + batch_size: int = 64, + lr: float = 1e-4, + weight_decay: float = 1e-5, + epochs: int = 50, + patience: int = 10, + # 随机种子 + seed: int = 42, + # 设备 + device: str = "cuda" if torch.cuda.is_available() else "cpu", +): + """ + 在外部 LiON 数据上预训练 backbone + delivery head。 + + 需要先运行 `make data_pretrain` 生成 parquet 文件。 + + 使用 --use-mpnn 启用 MPNN encoder（自动从 models/mpnn/all_amine_split_for_LiON 加载）。 + + 产出: + - models/pretrain_delivery.pt: 包含 backbone + delivery head 权重 + - models/pretrain_history.json: 训练历史 + """ + set_global_seed(seed) + logger.info(f"Using device: {device} | seed: {seed}") + device_obj = torch.device(device) + + # 加载已处理的 parquet 文件 + logger.info(f"Loading train data from {train_path}") + train_df = pd.read_parquet(train_path) + train_dataset = ExternalDeliveryDataset(train_df) + + logger.info(f"Loading val data from {val_path}") + val_df = pd.read_parquet(val_path) + val_dataset = ExternalDeliveryDataset(val_df) + + logger.info(f"Train samples: {len(train_dataset)}, Val samples: {len(val_dataset)}") + + train_loader = DataLoader( + train_dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_fn + ) + val_loader = DataLoader( + val_dataset, batch_size=batch_size, shuffle=False, collate_fn=collate_fn + ) + + # 解析 MPNN 配置 + # 优先级：mpnn_checkpoint > mpnn_ensemble_paths > use_mpnn（自动查找） + ensemble_paths_list = None + if mpnn_ensemble_paths: + ensemble_paths_list = mpnn_ensemble_paths.split(",") + elif use_mpnn and mpnn_checkpoint is None: + # --use-mpnn 但没有指定具体路径，自动查找 + logger.info(f"Auto-detecting MPNN ensemble from {DEFAULT_MPNN_ENSEMBLE_DIR}") + ensemble_paths_list = find_mpnn_ensemble_paths() + logger.info(f"Found {len(ensemble_paths_list)} MPNN models") + + enable_mpnn = mpnn_checkpoint is not None or ensemble_paths_list is not None + + # 创建模型 + logger.info( + f"Creating model (use_mpnn={enable_mpnn}, use_moe={use_moe}, use_llm={use_llm})..." + ) + common_kwargs = dict( + molformer_emb_path=molformer_emb, + molformer_smiles_csv=molformer_csv, + graphmvp_emb_path=graphmvp_emb, + graphmvp_smiles_csv=graphmvp_csv, + grover_emb_path=grover_emb, + grover_smiles_csv=grover_csv, + d_model=d_model, + num_heads=num_heads, + n_attn_layers=n_attn_layers, + set_transformer_block=set_transformer_block, + fusion_strategy=fusion_strategy, + head_hidden_dim=head_hidden_dim, + dropout=dropout, + use_moe=use_moe, + moe_n_experts=moe_n_experts, + moe_top_k=moe_top_k, + moe_expert_hidden_mult=moe_expert_hidden_mult, + moe_jitter_noise=moe_jitter_noise, + use_llm=use_llm, + llm_model_path=llm_model_path, + llm_freeze=llm_freeze, + llm_use_lora=llm_use_lora, + llm_lora_r=llm_lora_r, + llm_lora_alpha=llm_lora_alpha, + llm_lora_dropout=llm_lora_dropout, + ) + if enable_mpnn: + model = LNPModel( + mpnn_checkpoint=mpnn_checkpoint, + mpnn_ensemble_paths=ensemble_paths_list, + mpnn_device=mpnn_device, + **common_kwargs, + ) + else: + model = LNPModelWithoutMPNN(**common_kwargs) + + n_params_total = sum(p.numel() for p in model.parameters()) + n_params_trainable = sum(p.numel() for p in model.parameters() if p.requires_grad) + logger.info(f"Model parameters: {n_params_total:,} total, {n_params_trainable:,} trainable") + + # 预热 RDKit 缓存（避免训练时阻塞） + all_smiles = train_df["smiles"].tolist() + val_df["smiles"].tolist() + warmup_cache(model, all_smiles, batch_size=256) + + # 预训练 + logger.info("Starting pretraining on external data (delivery only)...") + result = pretrain( + train_loader=train_loader, + val_loader=val_loader, + model=model, + device=device_obj, + lr=lr, + weight_decay=weight_decay, + epochs=epochs, + patience=patience, + ) + + # 保存预训练 checkpoint + output_dir.mkdir(parents=True, exist_ok=True) + checkpoint_path = output_dir / "pretrain_delivery.pt" + torch.save( + { + "model_state_dict": model.state_dict(), + "backbone_state_dict": model.get_backbone_state_dict(), + "delivery_head_state_dict": model.get_delivery_head_state_dict(), + "config": { + "d_model": d_model, + "num_heads": num_heads, + "n_attn_layers": n_attn_layers, + "set_transformer_block": set_transformer_block, + "fusion_strategy": fusion_strategy, + "head_hidden_dim": head_hidden_dim, + "dropout": dropout, + "use_mpnn": enable_mpnn, + "use_moe": use_moe, + "molformer_emb": str(molformer_emb) if molformer_emb else None, + "molformer_csv": str(molformer_csv) if molformer_csv else None, + "graphmvp_emb": str(graphmvp_emb) if graphmvp_emb else None, + "graphmvp_csv": str(graphmvp_csv) if graphmvp_csv else None, + "grover_emb": str(grover_emb) if grover_emb else None, + "grover_csv": str(grover_csv) if grover_csv else None, + "moe_n_experts": moe_n_experts, + "moe_top_k": moe_top_k, + "moe_expert_hidden_mult": moe_expert_hidden_mult, + "moe_jitter_noise": moe_jitter_noise, + "use_llm": use_llm, + "llm_model_path": llm_model_path, + "llm_freeze": llm_freeze, + "llm_use_lora": llm_use_lora, + "llm_lora_r": llm_lora_r, + "llm_lora_alpha": llm_lora_alpha, + "llm_lora_dropout": llm_lora_dropout, + }, + "best_val_loss": result["best_val_loss"], + }, + checkpoint_path, + ) + logger.success(f"Saved pretrain checkpoint to {checkpoint_path}") + + # 保存训练历史 + history_path = output_dir / "pretrain_history.json" + with open(history_path, "w") as f: + json.dump(result["history"], f, indent=2) + logger.success(f"Saved pretrain history to {history_path}") + + # 绘制 loss 曲线图 + loss_plot_path = output_dir / "pretrain_loss_curves.png" + plot_loss_curves( + history=result["history"], + output_path=loss_plot_path, + title="Pretrain Loss Curves (Delivery)", + ) + logger.success(f"Saved loss curves plot to {loss_plot_path}") + + logger.success( + f"Pretraining complete! Best val_loss: {result['best_val_loss']:.4f}" + ) + + +@app.command() +def test( + # 数据路径 + val_path: Path = PROCESSED_DATA_DIR / "val_pretrain.parquet", + model_path: Path = MODELS_DIR / "pretrain_delivery.pt", + output_path: Path = MODELS_DIR / "pretrain_test_results.json", + # MPNN 参数 + use_mpnn: bool = False, + mpnn_device: str = "cpu", + # 其他参数 + batch_size: int = 64, + device: str = "cuda" if torch.cuda.is_available() else "cpu", +): + """ + 评估 pretrain 模型在外部数据上的 delivery 预测性能。 + + 输出详细指标：MSE, RMSE, MAE, R² + """ + import numpy as np + from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score + + logger.info(f"Using device: {device}") + device_obj = torch.device(device) + + # 加载模型 + logger.info(f"Loading pretrain model from {model_path}") + checkpoint = torch.load(model_path, map_location=device_obj, weights_only=False) + config = checkpoint["config"] + + # 解析 MPNN 配置 + enable_mpnn = config.get("use_mpnn", False) + common_kwargs = dict( + d_model=config["d_model"], + num_heads=config["num_heads"], + n_attn_layers=config["n_attn_layers"], + set_transformer_block=config.get("set_transformer_block", "sab"), + fusion_strategy=config["fusion_strategy"], + head_hidden_dim=config["head_hidden_dim"], + dropout=config["dropout"], + use_moe=config.get("use_moe", False), + molformer_emb_path=config.get("molformer_emb"), + molformer_smiles_csv=config.get("molformer_csv"), + graphmvp_emb_path=config.get("graphmvp_emb"), + graphmvp_smiles_csv=config.get("graphmvp_csv"), + grover_emb_path=config.get("grover_emb"), + grover_smiles_csv=config.get("grover_csv"), + moe_n_experts=config.get("moe_n_experts", 4), + moe_top_k=config.get("moe_top_k", 2), + moe_expert_hidden_mult=config.get("moe_expert_hidden_mult", 2), + moe_jitter_noise=config.get("moe_jitter_noise", 0.0), + use_llm=config.get("use_llm", False), + llm_model_path=config.get("llm_model_path", DEFAULT_MOLT5_PATH), + llm_freeze=config.get("llm_freeze", True), + llm_use_lora=config.get("llm_use_lora", False), + llm_lora_r=config.get("llm_lora_r", 8), + llm_lora_alpha=config.get("llm_lora_alpha", 16), + llm_lora_dropout=config.get("llm_lora_dropout", 0.05), + ) + if enable_mpnn or use_mpnn: + logger.info(f"Auto-detecting MPNN ensemble from {DEFAULT_MPNN_ENSEMBLE_DIR}") + ensemble_paths = find_mpnn_ensemble_paths() + logger.info(f"Found {len(ensemble_paths)} MPNN models") + model = LNPModel( + mpnn_ensemble_paths=ensemble_paths, + mpnn_device=mpnn_device, + **common_kwargs, + ) + else: + model = LNPModelWithoutMPNN(**common_kwargs) + + model.load_state_dict(checkpoint["model_state_dict"], strict=False) + model.to(device_obj) + model.eval() + + logger.info(f"Model config: {config}") + logger.info(f"Best val_loss from training: {checkpoint.get('best_val_loss', 'N/A')}") + + # 加载数据 + logger.info(f"Loading validation data from {val_path}") + val_df = pd.read_parquet(val_path) + val_dataset = ExternalDeliveryDataset(val_df) + val_loader = DataLoader( + val_dataset, batch_size=batch_size, shuffle=False, collate_fn=collate_fn + ) + logger.info(f"Validation samples: {len(val_dataset)}") + + # 预测 + logger.info("Running predictions...") + all_preds = [] + all_targets = [] + all_masks = [] + + with torch.no_grad(): + for batch in tqdm(val_loader, desc="Predicting"): + smiles = batch["smiles"] + tabular = {k: v.to(device_obj) for k, v in batch["tabular"].items()} + targets = batch["targets"]["delivery"].numpy() + mask = batch["mask"]["delivery"].numpy() + + pred = model.forward_delivery(smiles, tabular).squeeze(-1).cpu().numpy() + + all_preds.extend(pred) + all_targets.extend(targets) + all_masks.extend(mask) + + # 转为数组 + all_preds = np.array(all_preds) + all_targets = np.array(all_targets) + all_masks = np.array(all_masks, dtype=bool) + + # 只计算有效样本 + y_pred = all_preds[all_masks] + y_true = all_targets[all_masks] + + # 计算指标 + mse = float(mean_squared_error(y_true, y_pred)) + rmse = float(np.sqrt(mse)) + mae = float(mean_absolute_error(y_true, y_pred)) + r2 = float(r2_score(y_true, y_pred)) + + # 额外统计 + correlation = float(np.corrcoef(y_true, y_pred)[0, 1]) + + results = { + "model_path": str(model_path), + "val_path": str(val_path), + "n_samples": int(all_masks.sum()), + "metrics": { + "mse": mse, + "rmse": rmse, + "mae": mae, + "r2": r2, + "correlation": correlation, + }, + "statistics": { + "y_true_mean": float(y_true.mean()), + "y_true_std": float(y_true.std()), + "y_pred_mean": float(y_pred.mean()), + "y_pred_std": float(y_pred.std()), + } + } + + # 打印结果 + logger.info("\n" + "=" * 50) + logger.info("PRETRAIN MODEL EVALUATION (Delivery)") + logger.info("=" * 50) + logger.info(f"Samples: {results['n_samples']}") + logger.info("\n[Metrics]") + logger.info(f" MSE: {mse:.4f}") + logger.info(f" RMSE: {rmse:.4f}") + logger.info(f" MAE: {mae:.4f}") + logger.info(f" R²: {r2:.4f}") + logger.info(f" Correlation: {correlation:.4f}") + logger.info("\n[Statistics]") + logger.info(f" True: mean={y_true.mean():.4f}, std={y_true.std():.4f}") + logger.info(f" Pred: mean={y_pred.mean():.4f}, std={y_pred.std():.4f}") + + # 保存结果 + with open(output_path, "w") as f: + json.dump(results, f, indent=2) + logger.success(f"\nSaved results to {output_path}") + + +if __name__ == "__main__": + app() + diff --git a/results/pretrained_encoders/baseline_pretrain_s42_summary.json b/results/pretrained_encoders/baseline_pretrain_s42_summary.json new file mode 100644 index 0000000..a68e516 --- /dev/null +++ b/results/pretrained_encoders/baseline_pretrain_s42_summary.json @@ -0,0 +1,932 @@ +{ + "fold_results": [ + { + "fold": 0, + "best_params": { + "dropout": 0.3980773018180746, + "lr": 0.0009896030589860336, + "weight_decay": 0.0002988554096805342, + "backbone_lr_ratio": 0.2882797738700643, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 14, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.601088294900837, + "rmse": 1.265341177272295, + "mae": 0.5760945507650634, + "r2": 0.02009455288902051 + }, + "delivery": { + "n_samples": 58, + "mse": 0.569556123704086, + "rmse": 0.7546894220168228, + "mae": 0.5891120558411911, + "r2": 0.2760597975244864 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7738095238095238, + "precision": 0.7194960212201591, + "recall": 0.7359230220955096, + "f1": 0.7262905162064826 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7261904761904762, + "precision": 0.684408546477512, + "recall": 0.7380952380952381, + "f1": 0.6942196418797404 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9655172413793104, + "precision": 0.75, + "recall": 0.9821428571428572, + "f1": 0.8242424242424242 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.2787090798576488, + "js_divergence": 0.05423684052771057 + } + } + }, + { + "fold": 1, + "best_params": { + "dropout": 0.23085562232445592, + "lr": 0.0005227270589225132, + "weight_decay": 0.00461701039547356, + "backbone_lr_ratio": 0.12087164274488635, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 11, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.3552515818625956, + "rmse": 0.5960298498083763, + "mae": 0.4518711648970133, + "r2": -0.08481203101542345 + }, + "delivery": { + "n_samples": 61, + "mse": 1.078371115810122, + "rmse": 1.038446491548853, + "mae": 0.6491757012354057, + "r2": 0.18375080395351728 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6547619047619048, + "precision": 0.5583333333333333, + "recall": 0.5634920634920635, + "f1": 0.5598915989159892 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5714285714285714, + "precision": 0.5153903903903904, + "recall": 0.5209076637648066, + "f1": 0.507505844715147 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9508196721311475, + "precision": 0.75, + "recall": 0.9741379310344828, + "f1": 0.8200589970501475 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.5193434798072222, + "js_divergence": 0.12637397289897973 + } + } + }, + { + "fold": 2, + "best_params": { + "dropout": 0.318342405407684, + "lr": 0.0008937362022799393, + "weight_decay": 0.06633867694372529, + "backbone_lr_ratio": 0.20906844321970927, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 10, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.3646369784573562, + "rmse": 0.6038517851736105, + "mae": 0.47954003302203047, + "r2": 0.2798484825035997 + }, + "delivery": { + "n_samples": 60, + "mse": 0.6872133280793995, + "rmse": 0.828983309891942, + "mae": 0.5440382711666947, + "r2": 0.08519994983143697 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7380952380952381, + "precision": 0.6785714285714286, + "recall": 0.7052785923753666, + "f1": 0.6867796610169492 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6547619047619048, + "precision": 0.6053846153846153, + "recall": 0.6720354808590102, + "f1": 0.6203502415458937 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.35680852482645714, + "js_divergence": 0.09025645010360116 + } + } + }, + { + "fold": 3, + "best_params": { + "dropout": 0.12612237693012526, + "lr": 0.0009464809649969671, + "weight_decay": 0.07051438027639334, + "backbone_lr_ratio": 0.9343703462702573, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 19, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.6485083926359867, + "rmse": 1.2839425192102591, + "mae": 0.5286768473437932, + "r2": 0.06759108835091887 + }, + "delivery": { + "n_samples": 59, + "mse": 0.6919949158910226, + "rmse": 0.8318623178693831, + "mae": 0.5334780438945202, + "r2": 0.3272987252110757 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7142857142857143, + "precision": 0.6375, + "recall": 0.6451612903225806, + "f1": 0.6407697790449038 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5952380952380952, + "precision": 0.5380952380952381, + "recall": 0.5259426847662142, + "f1": 0.5062561094819159 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9333333333333333, + "precision": 0.7142857142857143, + "recall": 0.9649122807017544, + "f1": 0.7818181818181817 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.2642989361691452, + "js_divergence": 0.06629497269911663 + } + } + }, + { + "fold": 4, + "best_params": { + "dropout": 0.20496607857127885, + "lr": 0.0009573095972231165, + "weight_decay": 0.00038069869878401156, + "backbone_lr_ratio": 0.04121720375781922, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 17, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.7197053265910517, + "rmse": 0.8483544816826583, + "mae": 0.511052241144214, + "r2": 0.036944874168868824 + }, + "delivery": { + "n_samples": 58, + "mse": 1.0001767893259463, + "rmse": 1.0000883907565103, + "mae": 0.60194506123662, + "r2": 0.03284555574221559 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7261904761904762, + "precision": 0.6836007130124777, + "recall": 0.7265395894428153, + "f1": 0.6891391794046662 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6904761904761905, + "precision": 0.6314814814814814, + "recall": 0.6444444444444445, + "f1": 0.6326984475152934 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.865909090909091 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.2786833611146747, + "js_divergence": 0.07839598379605058 + } + } + }, + { + "fold": 0, + "best_params": { + "dropout": 0.3980773018180746, + "lr": 0.0009896030589860336, + "weight_decay": 0.0002988554096805342, + "backbone_lr_ratio": 0.2882797738700643, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 14, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.5048982722984092, + "rmse": 1.2267429528219875, + "mae": 0.5805474416723093, + "r2": 0.07896521442969773 + }, + "delivery": { + "n_samples": 58, + "mse": 0.6736236117326778, + "rmse": 0.8207457655892462, + "mae": 0.6456835756142592, + "r2": 0.1437837404002562 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7857142857142857, + "precision": 0.7305773342836779, + "recall": 0.7305773342836779, + "f1": 0.7305773342836779 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6646103896103895, + "recall": 0.7014285714285715, + "f1": 0.6618818596621038 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9827586206896551, + "precision": 0.9912280701754386, + "recall": 0.75, + "f1": 0.8289085545722714 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.2890762442571175, + "js_divergence": 0.05687567953491692 + } + } + }, + { + "fold": 1, + "best_params": { + "dropout": 0.23085562232445592, + "lr": 0.0005227270589225132, + "weight_decay": 0.00461701039547356, + "backbone_lr_ratio": 0.12087164274488635, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 11, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.34313442749289724, + "rmse": 0.5857767727495665, + "mae": 0.44384193185362075, + "r2": -0.04781054949350838 + }, + "delivery": { + "n_samples": 61, + "mse": 1.099845523641604, + "rmse": 1.0487352018701404, + "mae": 0.6761384357499783, + "r2": 0.1674962252922061 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6547619047619048, + "precision": 0.6215277777777778, + "recall": 0.6587301587301587, + "f1": 0.6149834044570887 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5595238095238095, + "precision": 0.5017255158384191, + "recall": 0.5057561486132914, + "f1": 0.4936173861292259 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9508196721311475, + "precision": 0.75, + "recall": 0.9741379310344828, + "f1": 0.8200589970501475 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.5199198163542676, + "js_divergence": 0.12515294319531267 + } + } + }, + { + "fold": 2, + "best_params": { + "dropout": 0.318342405407684, + "lr": 0.0008937362022799393, + "weight_decay": 0.06633867694372529, + "backbone_lr_ratio": 0.20906844321970927, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 10, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.4877212975355261, + "rmse": 0.6983704586646876, + "mae": 0.5448984429905457, + "r2": 0.036759151467687046 + }, + "delivery": { + "n_samples": 60, + "mse": 0.7067592342133922, + "rmse": 0.8406897371880974, + "mae": 0.5448016864635671, + "r2": 0.05918096099439174 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7380952380952381, + "precision": 0.6851851851851851, + "recall": 0.7199413489736071, + "f1": 0.6936339522546419 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6240783117213323, + "recall": 0.6851073762838468, + "f1": 0.6414141414141414 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.38840109926844824, + "js_divergence": 0.09646626485112685 + } + } + }, + { + "fold": 3, + "best_params": { + "dropout": 0.12612237693012526, + "lr": 0.0009464809649969671, + "weight_decay": 0.07051438027639334, + "backbone_lr_ratio": 0.9343703462702573, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 19, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.9244709959119324, + "rmse": 1.3872530396117113, + "mae": 0.5739306664004563, + "r2": -0.08849546342266934 + }, + "delivery": { + "n_samples": 59, + "mse": 0.7217060909432494, + "rmse": 0.849532866311392, + "mae": 0.5202912685256136, + "r2": 0.2984159330486872 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7023809523809523, + "precision": 0.6267796610169492, + "recall": 0.6370967741935484, + "f1": 0.6307367680675224 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6190476190476191, + "precision": 0.5577586206896551, + "recall": 0.5861990950226245, + "f1": 0.5614861329147044 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9333333333333333, + "precision": 0.7142857142857143, + "recall": 0.9649122807017544, + "f1": 0.7818181818181817 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.2617822353923735, + "js_divergence": 0.06083979574849321 + } + } + }, + { + "fold": 4, + "best_params": { + "dropout": 0.20496607857127885, + "lr": 0.0009573095972231165, + "weight_decay": 0.00038069869878401156, + "backbone_lr_ratio": 0.04121720375781922, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 17, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.7148724493029326, + "rmse": 0.8455013005920999, + "mae": 0.5005925307438398, + "r2": 0.04341186430062394 + }, + "delivery": { + "n_samples": 58, + "mse": 1.0380505423911823, + "rmse": 1.018847654161888, + "mae": 0.6400383858588236, + "r2": -0.0037777382480910227 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7738095238095238, + "precision": 0.7213166144200627, + "recall": 0.7587976539589443, + "f1": 0.7325289089994973 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6904761904761905, + "precision": 0.6308992939427722, + "recall": 0.6444444444444445, + "f1": 0.633663610371685 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.865909090909091 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.2701739757789761, + "js_divergence": 0.07372463442305974 + } + } + }, + { + "fold": 0, + "best_params": { + "dropout": 0.3980773018180746, + "lr": 0.0009896030589860336, + "weight_decay": 0.0002988554096805342, + "backbone_lr_ratio": 0.2882797738700643, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 14, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.4541105583695868, + "rmse": 1.2058650664023678, + "mae": 0.5784477213150766, + "r2": 0.11004854548874443 + }, + "delivery": { + "n_samples": 58, + "mse": 0.59667434347555, + "rmse": 0.7724469842491134, + "mae": 0.6142617367079546, + "r2": 0.24159090377534476 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.75, + "precision": 0.6916445623342176, + "recall": 0.7059871703492516, + "f1": 0.6974789915966386 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7142857142857143, + "precision": 0.6695636813283872, + "recall": 0.7142857142857144, + "f1": 0.6750809541465826 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9655172413793104, + "precision": 0.75, + "recall": 0.9821428571428572, + "f1": 0.8242424242424242 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.2587242230390428, + "js_divergence": 0.052354750762398025 + } + } + }, + { + "fold": 1, + "best_params": { + "dropout": 0.23085562232445592, + "lr": 0.0005227270589225132, + "weight_decay": 0.00461701039547356, + "backbone_lr_ratio": 0.12087164274488635, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 11, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.36298737292148275, + "rmse": 0.6024843341710079, + "mae": 0.45925771394034937, + "r2": -0.1084343866601285 + }, + "delivery": { + "n_samples": 61, + "mse": 1.0462031939942293, + "rmse": 1.022840747132333, + "mae": 0.634948962077987, + "r2": 0.20809960181702725 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7023809523809523, + "precision": 0.6514423076923077, + "recall": 0.6904761904761905, + "f1": 0.6554552912223134 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5476190476190477, + "precision": 0.48089218250508575, + "recall": 0.48011512297226583, + "f1": 0.47062888038209943 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9508196721311475, + "precision": 0.75, + "recall": 0.9741379310344828, + "f1": 0.8200589970501475 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.5244687341232338, + "js_divergence": 0.12337098779147845 + } + } + }, + { + "fold": 2, + "best_params": { + "dropout": 0.318342405407684, + "lr": 0.0008937362022799393, + "weight_decay": 0.06633867694372529, + "backbone_lr_ratio": 0.20906844321970927, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 10, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.4579348623566749, + "rmse": 0.6767088460753818, + "mae": 0.5252955685635763, + "r2": 0.09558682875266244 + }, + "delivery": { + "n_samples": 60, + "mse": 0.6918557253424359, + "rmse": 0.8317786516510483, + "mae": 0.5500254434921469, + "r2": 0.07902011443593149 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6428571428571428, + "recall": 0.6796187683284457, + "f1": 0.6415362731152205 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6222222222222222, + "recall": 0.6851073762838468, + "f1": 0.636502849002849 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.3340915841461506, + "js_divergence": 0.08509576023385822 + } + } + }, + { + "fold": 3, + "best_params": { + "dropout": 0.12612237693012526, + "lr": 0.0009464809649969671, + "weight_decay": 0.07051438027639334, + "backbone_lr_ratio": 0.9343703462702573, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 19, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.8202166309492056, + "rmse": 1.3491540427057267, + "mae": 0.5770047424306408, + "r2": -0.02952839998289769 + }, + "delivery": { + "n_samples": 59, + "mse": 0.8240920088267009, + "rmse": 0.9077951359347002, + "mae": 0.5719114731271135, + "r2": 0.1988846563023151 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7142857142857143, + "precision": 0.6375, + "recall": 0.6451612903225806, + "f1": 0.6407697790449038 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5833333333333334, + "precision": 0.475235446249939, + "recall": 0.4788838612368025, + "f1": 0.4725999853769101 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9333333333333333, + "precision": 0.7142857142857143, + "recall": 0.9649122807017544, + "f1": 0.7818181818181817 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.26692129352633476, + "js_divergence": 0.061695485072966134 + } + } + }, + { + "fold": 4, + "best_params": { + "dropout": 0.20496607857127885, + "lr": 0.0009573095972231165, + "weight_decay": 0.00038069869878401156, + "backbone_lr_ratio": 0.04121720375781922, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 17, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.713518768290232, + "rmse": 0.844700401497615, + "mae": 0.5241211098341626, + "r2": 0.045223257644336656 + }, + "delivery": { + "n_samples": 58, + "mse": 1.0365573602046312, + "rmse": 1.018114610544722, + "mae": 0.6499280054764501, + "r2": -0.0023338557232996937 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.75, + "precision": 0.6949843260188088, + "recall": 0.7280058651026393, + "f1": 0.7043740573152337 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7023809523809523, + "precision": 0.639751552795031, + "recall": 0.6722222222222222, + "f1": 0.6522525093953665 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.865909090909091 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.3119448408030975, + "js_divergence": 0.08658354575321366 + } + } + } + ], + "summary_stats": { + "size": { + "mse_mean": 0.9648704139917805, + "mse_std": 0.5891656318820191, + "rmse_mean": 0.9346718018959568, + "rmse_std": 0.3020911066753635, + "mae_mean": 0.5236781804611128, + "mae_std": 0.047009951044540414, + "r2_mean": 0.030359535294768855, + "r2_std": 0.09435084837045356 + }, + "delivery": { + "mse_mean": 0.8308453271717486, + "mse_std": 0.18767747504329296, + "rmse_mean": 0.9057064857810794, + "rmse_std": 0.10266980464496812, + "mae_mean": 0.5977185404312217, + "mae_std": 0.049065196777405395, + "r2_mean": 0.1530343582905001, + "r2_std": 0.10421669155896922 + }, + "pdi": { + "accuracy_mean": 0.7238095238095238, + "accuracy_std": 0.039411774660111545, + "precision_mean": 0.665421093848235, + "precision_std": 0.0446473083216482, + "recall_mean": 0.6887191408298253, + "recall_std": 0.04935509276909138, + "f1_mean": 0.669663032996382, + "f1_std": 0.04712559651378041 + }, + "ee": { + "accuracy_mean": 0.6460317460317461, + "accuracy_std": 0.058558307129195915, + "precision_mean": 0.5894331659154981, + "precision_std": 0.06893389274036124, + "recall_mean": 0.6169983629815563, + "recall_std": 0.08823505436531638, + "f1_mean": 0.5906772395955772, + "f1_std": 0.07683138902833272 + }, + "toxic": { + "accuracy_mean": 0.9511890595408049, + "accuracy_std": 0.016071213375949578, + "precision_mean": 0.7617961570593149, + "precision_std": 0.0689235340767993, + "recall_mean": 0.958294443004062, + "recall_std": 0.05613748087593979, + "f1_mean": 0.8151453855878633, + "f1_std": 0.03136647604932657 + }, + "biodist": { + "kl_divergence_mean": 0.34155649523094606, + "kl_divergence_std": 0.09686930050549591, + "js_divergence_mean": 0.08251453782615217, + "js_divergence_std": 0.024921020774773404 + } + } +} \ No newline at end of file diff --git a/results/pretrained_encoders/graphmvp_pretrain_s42_summary.json b/results/pretrained_encoders/graphmvp_pretrain_s42_summary.json new file mode 100644 index 0000000..1614ce7 --- /dev/null +++ b/results/pretrained_encoders/graphmvp_pretrain_s42_summary.json @@ -0,0 +1,932 @@ +{ + "fold_results": [ + { + "fold": 0, + "best_params": { + "dropout": 0.3980773018180746, + "lr": 0.0009896030589860336, + "weight_decay": 0.0002988554096805342, + "backbone_lr_ratio": 0.2882797738700643, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 15, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.5221819405605161, + "rmse": 1.2337673770044806, + "mae": 0.5776655403784958, + "r2": 0.06838718401748567 + }, + "delivery": { + "n_samples": 58, + "mse": 0.614112021357094, + "rmse": 0.7836529980527696, + "mae": 0.6216259215926302, + "r2": 0.21942656293011076 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7976190476190477, + "precision": 0.7458333333333333, + "recall": 0.7523164647184604, + "f1": 0.7489010022859153 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6691176470588235, + "recall": 0.7114285714285714, + "f1": 0.6657848324514991 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9655172413793104, + "precision": 0.75, + "recall": 0.9821428571428572, + "f1": 0.8242424242424242 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.2940147973370504, + "js_divergence": 0.06236386129092423 + } + } + }, + { + "fold": 1, + "best_params": { + "dropout": 0.2902385468029013, + "lr": 0.00018739040729381568, + "weight_decay": 0.08181191083032673, + "backbone_lr_ratio": 0.6566372495858538, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 12, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.2934246330317906, + "rmse": 0.5416868403716215, + "mae": 0.4069202210133274, + "r2": 0.10398548983157396 + }, + "delivery": { + "n_samples": 61, + "mse": 1.156314335236939, + "rmse": 1.0753205732417375, + "mae": 0.6882066883329974, + "r2": 0.12475340569084314 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6547619047619048, + "precision": 0.5583333333333333, + "recall": 0.5634920634920635, + "f1": 0.5598915989159892 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5595238095238095, + "precision": 0.48693693693693696, + "recall": 0.49312592169735026, + "f1": 0.47630082959336323 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9508196721311475, + "precision": 0.75, + "recall": 0.9741379310344828, + "f1": 0.8200589970501475 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.5092761936607958, + "js_divergence": 0.13355010688479413 + } + } + }, + { + "fold": 2, + "best_params": { + "dropout": 0.18117503720006686, + "lr": 8.135394049241399e-05, + "weight_decay": 0.015076469897649124, + "backbone_lr_ratio": 0.9586175198423679, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 18, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.34434236619012065, + "rmse": 0.586806924115693, + "mae": 0.46669395072967745, + "r2": 0.3199299791282213 + }, + "delivery": { + "n_samples": 60, + "mse": 0.6575735692519624, + "rmse": 0.810909100486585, + "mae": 0.55640976827126, + "r2": 0.12465560610938209 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6190476190476191, + "precision": 0.6190476190476191, + "recall": 0.653958944281525, + "f1": 0.5961538461538461 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6666666666666666, + "precision": 0.6166666666666667, + "recall": 0.6785714285714285, + "f1": 0.6334814122048165 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.41784407477442437, + "js_divergence": 0.11469543309672324 + } + } + }, + { + "fold": 3, + "best_params": { + "dropout": 0.1799936812783694, + "lr": 0.00033865324369584493, + "weight_decay": 0.002833833442807743, + "backbone_lr_ratio": 0.3278334190266754, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 8, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.6764751937886317, + "rmse": 1.2947877022078298, + "mae": 0.622077302143247, + "r2": 0.05177285245868457 + }, + "delivery": { + "n_samples": 59, + "mse": 0.725000410122951, + "rmse": 0.8514695591287753, + "mae": 0.6181600616487154, + "r2": 0.2952134634049711 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6428571428571429, + "precision": 0.5714285714285714, + "recall": 0.5821114369501467, + "f1": 0.5728813559322035 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6666666666666666, + "precision": 0.5873015873015873, + "recall": 0.6022121669180492, + "f1": 0.5872162485065711 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9333333333333333, + "precision": 0.7142857142857143, + "recall": 0.9649122807017544, + "f1": 0.7818181818181817 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.4920884182489342, + "js_divergence": 0.12950821301606347 + } + } + }, + { + "fold": 4, + "best_params": { + "dropout": 0.19398015625011103, + "lr": 0.0008450080181209037, + "weight_decay": 0.002901987198512681, + "backbone_lr_ratio": 0.5310897731873246, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 16, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.7301598129307739, + "rmse": 0.8544938928575054, + "mae": 0.4922567930327551, + "r2": 0.022955472832782875 + }, + "delivery": { + "n_samples": 58, + "mse": 1.125780851035229, + "rmse": 1.0610282046370063, + "mae": 0.6240321383650961, + "r2": -0.08861149844600491 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.75, + "precision": 0.683728813559322, + "recall": 0.6986803519061584, + "f1": 0.6898188851767189 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6666666666666666, + "precision": 0.6060606060606061, + "recall": 0.6418803418803419, + "f1": 0.6178929765886287 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.865909090909091 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.26640631927709985, + "js_divergence": 0.07248344781159008 + } + } + }, + { + "fold": 0, + "best_params": { + "dropout": 0.3980773018180746, + "lr": 0.0009896030589860336, + "weight_decay": 0.0002988554096805342, + "backbone_lr_ratio": 0.2882797738700643, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 15, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.5188104198487684, + "rmse": 1.2324002677088188, + "mae": 0.5873989220259779, + "r2": 0.07045063768272763 + }, + "delivery": { + "n_samples": 58, + "mse": 0.6666016049923045, + "rmse": 0.8164567379796094, + "mae": 0.6511167181206161, + "r2": 0.15270913470266467 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7857142857142857, + "precision": 0.7305773342836779, + "recall": 0.7305773342836779, + "f1": 0.7305773342836779 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7023809523809523, + "precision": 0.691027926322044, + "recall": 0.741904761904762, + "f1": 0.6917852553884204 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9827586206896551, + "precision": 0.9912280701754386, + "recall": 0.75, + "f1": 0.8289085545722714 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.2601734547154161, + "js_divergence": 0.056514013464432016 + } + } + }, + { + "fold": 1, + "best_params": { + "dropout": 0.2902385468029013, + "lr": 0.00018739040729381568, + "weight_decay": 0.08181191083032673, + "backbone_lr_ratio": 0.6566372495858538, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 12, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.3231063887830616, + "rmse": 0.5684244794016717, + "mae": 0.42929340717160985, + "r2": 0.01334796030441776 + }, + "delivery": { + "n_samples": 61, + "mse": 1.1342861224694418, + "rmse": 1.0650286956084525, + "mae": 0.6788244451106084, + "r2": 0.14142717476551148 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.5595238095238095, + "precision": 0.5477272727272727, + "recall": 0.5634920634920635, + "f1": 0.5238241152137275 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5476190476190477, + "precision": 0.473011973011973, + "recall": 0.4779744065458351, + "f1": 0.46420797118471535 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9508196721311475, + "precision": 0.75, + "recall": 0.9741379310344828, + "f1": 0.8200589970501475 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.5813128628991394, + "js_divergence": 0.14714358123767265 + } + } + }, + { + "fold": 2, + "best_params": { + "dropout": 0.18117503720006686, + "lr": 8.135394049241399e-05, + "weight_decay": 0.015076469897649124, + "backbone_lr_ratio": 0.9586175198423679, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 18, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.43267246099702694, + "rmse": 0.6577784284977936, + "mae": 0.5142740875349513, + "r2": 0.14547962007547455 + }, + "delivery": { + "n_samples": 60, + "mse": 0.6198350807939489, + "rmse": 0.7872960566355892, + "mae": 0.5207243816383804, + "r2": 0.1748920752290377 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6904761904761905, + "precision": 0.6418269230769231, + "recall": 0.6730205278592376, + "f1": 0.6452241715399609 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6190476190476191, + "recall": 0.6757703081232492, + "f1": 0.63584229390681 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.42003529077921203, + "js_divergence": 0.11409932566058836 + } + } + }, + { + "fold": 3, + "best_params": { + "dropout": 0.1799936812783694, + "lr": 0.00033865324369584493, + "weight_decay": 0.002833833442807743, + "backbone_lr_ratio": 0.3278334190266754, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 8, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.709633019803758, + "rmse": 1.3075293571479603, + "mae": 0.629169082886214, + "r2": 0.03301853333873328 + }, + "delivery": { + "n_samples": 59, + "mse": 0.7678785210635485, + "rmse": 0.8762867801487983, + "mae": 0.6255631820472368, + "r2": 0.2535308451835053 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6666666666666666, + "precision": 0.5888594164456233, + "recall": 0.5982404692082112, + "f1": 0.5916666666666667 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6547619047619048, + "precision": 0.5874977416440831, + "recall": 0.6237556561085973, + "f1": 0.5958755389330425 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9333333333333333, + "precision": 0.7142857142857143, + "recall": 0.9649122807017544, + "f1": 0.7818181818181817 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.46770485035756304, + "js_divergence": 0.12378056507326861 + } + } + }, + { + "fold": 4, + "best_params": { + "dropout": 0.19398015625011103, + "lr": 0.0008450080181209037, + "weight_decay": 0.002901987198512681, + "backbone_lr_ratio": 0.5310897731873246, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 16, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.6133402094103233, + "rmse": 0.7831603982648275, + "mae": 0.4741146452669498, + "r2": 0.17927461319654014 + }, + "delivery": { + "n_samples": 58, + "mse": 1.0688240926391441, + "rmse": 1.0338394907523818, + "mae": 0.6684364553155571, + "r2": -0.0335352533249651 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7857142857142857, + "precision": 0.721875, + "recall": 0.7082111436950147, + "f1": 0.7142857142857142 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6428571428571429, + "precision": 0.5752393980848153, + "recall": 0.6188034188034188, + "f1": 0.5883517571357862 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.865909090909091 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.28068032170399054, + "js_divergence": 0.07478332635258625 + } + } + }, + { + "fold": 0, + "best_params": { + "dropout": 0.3980773018180746, + "lr": 0.0009896030589860336, + "weight_decay": 0.0002988554096805342, + "backbone_lr_ratio": 0.2882797738700643, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 15, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.3818212591739194, + "rmse": 1.1755089362373727, + "mae": 0.5521946805697607, + "r2": 0.1542913759905169 + }, + "delivery": { + "n_samples": 58, + "mse": 0.6972128277816507, + "rmse": 0.8349927112146852, + "mae": 0.6792561926333041, + "r2": 0.11380042333630946 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7857142857142857, + "precision": 0.7305773342836779, + "recall": 0.7305773342836779, + "f1": 0.7305773342836779 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7261904761904762, + "precision": 0.6891715244656421, + "recall": 0.7380952380952381, + "f1": 0.6965863304906686 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9655172413793104, + "precision": 0.75, + "recall": 0.9821428571428572, + "f1": 0.8242424242424242 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.30701952754304446, + "js_divergence": 0.057564049906523536 + } + } + }, + { + "fold": 1, + "best_params": { + "dropout": 0.2902385468029013, + "lr": 0.00018739040729381568, + "weight_decay": 0.08181191083032673, + "backbone_lr_ratio": 0.6566372495858538, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 12, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.34204974602902377, + "rmse": 0.5848501910994163, + "mae": 0.44300510672231513, + "r2": -0.04449831793169534 + }, + "delivery": { + "n_samples": 61, + "mse": 1.1297399798287129, + "rmse": 1.0628922710363045, + "mae": 0.6765872548166357, + "r2": 0.14486827701797667 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6428571428571429, + "precision": 0.5374198146828225, + "recall": 0.5396825396825397, + "f1": 0.5381231671554252 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5714285714285714, + "precision": 0.5202020202020202, + "recall": 0.5292564578278864, + "f1": 0.5119174942704355 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.5145521130605141, + "js_divergence": 0.1322571633039113 + } + } + }, + { + "fold": 2, + "best_params": { + "dropout": 0.18117503720006686, + "lr": 8.135394049241399e-05, + "weight_decay": 0.015076469897649124, + "backbone_lr_ratio": 0.9586175198423679, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 18, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.4742411460721128, + "rmse": 0.6886516870465887, + "mae": 0.5345059656759813, + "r2": 0.06338220975850639 + }, + "delivery": { + "n_samples": 60, + "mse": 0.6525032683185743, + "rmse": 0.807776744106052, + "mae": 0.5494959851183618, + "r2": 0.13140505545605896 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.5952380952380952, + "precision": 0.6319444444444444, + "recall": 0.6671554252199414, + "f1": 0.583673469387755 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6666666666666666, + "precision": 0.6091142490372272, + "recall": 0.669234360410831, + "f1": 0.6249493532803491 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.4235028564925827, + "js_divergence": 0.11443098458109038 + } + } + }, + { + "fold": 3, + "best_params": { + "dropout": 0.1799936812783694, + "lr": 0.00033865324369584493, + "weight_decay": 0.002833833442807743, + "backbone_lr_ratio": 0.3278334190266754, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 8, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.7043255314620636, + "rmse": 1.305498192822213, + "mae": 0.6349209440782307, + "r2": 0.036020489197966854 + }, + "delivery": { + "n_samples": 59, + "mse": 0.7865588440055382, + "rmse": 0.886881527604188, + "mae": 0.6117917100851566, + "r2": 0.23537135185779967 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6904761904761905, + "precision": 0.616710875331565, + "recall": 0.6290322580645161, + "f1": 0.6208333333333333 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6092796092796093, + "recall": 0.636827551533434, + "f1": 0.6157297695238403 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9333333333333333, + "precision": 0.7142857142857143, + "recall": 0.9649122807017544, + "f1": 0.7818181818181817 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.475069604298859, + "js_divergence": 0.12514035771620471 + } + } + }, + { + "fold": 4, + "best_params": { + "dropout": 0.19398015625011103, + "lr": 0.0008450080181209037, + "weight_decay": 0.002901987198512681, + "backbone_lr_ratio": 0.5310897731873246, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 16, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.8580511263094861, + "rmse": 0.9263104913091971, + "mae": 0.5776896519043172, + "r2": -0.14817899060384732 + }, + "delivery": { + "n_samples": 58, + "mse": 1.043085209273457, + "rmse": 1.0213154308407648, + "mae": 0.59220993094917, + "r2": -0.008646178010456307 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6904761904761905, + "precision": 0.6333333333333333, + "recall": 0.6583577712609971, + "f1": 0.6379310344827587 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7261904761904762, + "precision": 0.6652777777777779, + "recall": 0.7064102564102565, + "f1": 0.6809523809523809 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.865909090909091 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.24694979259516875, + "js_divergence": 0.06657100087180373 + } + } + } + ], + "summary_stats": { + "size": { + "mse_mean": 0.9283090169594252, + "mse_std": 0.5614727660610882, + "rmse_mean": 0.9161103444061993, + "rmse_std": 0.29841389684694647, + "mae_mean": 0.529478686742254, + "mae_std": 0.07272420920077967, + "r2_mean": 0.07130794061853929, + "r2_std": 0.10242823171449583 + }, + "delivery": { + "mse_mean": 0.8563537825446997, + "mse_std": 0.21315743069248957, + "rmse_mean": 0.91834312543158, + "rmse_std": 0.1140161677888585, + "mae_mean": 0.624162722269715, + "mae_std": 0.05009549555609884, + "r2_mean": 0.13208402972684966, + "r2_std": 0.10281086532328124 + }, + "pdi": { + "accuracy_mean": 0.6904761904761904, + "accuracy_std": 0.07299861769883757, + "precision_mean": 0.6372815612874345, + "precision_std": 0.06842608292617637, + "recall_mean": 0.6499270752265488, + "recall_std": 0.06582138762012522, + "f1_mean": 0.6322908686064913, + "f1_std": 0.07211332053746232 + }, + "ee": { + "accuracy_mean": 0.6555555555555556, + "accuracy_std": 0.05314498108820064, + "precision_mean": 0.6003302188598288, + "precision_std": 0.06446111150537735, + "recall_mean": 0.6363500564172834, + "recall_std": 0.07965961750314356, + "f1_mean": 0.6057916296274218, + "f1_std": 0.07002027044312158 + }, + "toxic": { + "accuracy_mean": 0.9500961633659415, + "accuracy_std": 0.01660762512329349, + "precision_mean": 0.7594152046783624, + "precision_std": 0.0698998626999633, + "recall_mean": 0.9577197303603838, + "recall_std": 0.05601633670665784, + "f1_mean": 0.8126176429273773, + "f1_std": 0.03238005389957549 + }, + "biodist": { + "kl_divergence_mean": 0.39710869851625297, + "kl_divergence_std": 0.10730729317255493, + "js_divergence_mean": 0.10165902868454511, + "js_divergence_std": 0.03125967849638632 + } + } +} \ No newline at end of file diff --git a/results/pretrained_encoders/grover_pretrain_s42_summary.json b/results/pretrained_encoders/grover_pretrain_s42_summary.json new file mode 100644 index 0000000..71f08d1 --- /dev/null +++ b/results/pretrained_encoders/grover_pretrain_s42_summary.json @@ -0,0 +1,932 @@ +{ + "fold_results": [ + { + "fold": 0, + "best_params": { + "dropout": 0.25888363054578495, + "lr": 0.0006536264082965735, + "weight_decay": 0.00932351457657158, + "backbone_lr_ratio": 0.1667894332719967, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 13, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.5789235054424788, + "rmse": 1.2565522294924627, + "mae": 0.5897082271031946, + "r2": 0.03365994962165875 + }, + "delivery": { + "n_samples": 58, + "mse": 0.5843462843532253, + "rmse": 0.7644254602989263, + "mae": 0.6106570679694414, + "r2": 0.2572606108431995 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7261904761904762, + "precision": 0.7065217391304348, + "recall": 0.7573057733428368, + "f1": 0.7039987743220468 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6388888888888888, + "recall": 0.6842857142857143, + "f1": 0.6433139534883722 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9655172413793104, + "precision": 0.75, + "recall": 0.9821428571428572, + "f1": 0.8242424242424242 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.295739533929628, + "js_divergence": 0.06360146142338208 + } + } + }, + { + "fold": 1, + "best_params": { + "dropout": 0.37015922545878527, + "lr": 0.0009352630241255291, + "weight_decay": 0.005516105285933501, + "backbone_lr_ratio": 0.12042458689005697, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 9, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.3504606454005027, + "rmse": 0.5919971667166176, + "mae": 0.4625389823867452, + "r2": -0.07018221434674188 + }, + "delivery": { + "n_samples": 61, + "mse": 1.0871146054096508, + "rmse": 1.0426478817940652, + "mae": 0.6687804595490948, + "r2": 0.17713261263549906 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6904761904761905, + "precision": 0.6067796610169491, + "recall": 0.6190476190476191, + "f1": 0.6108339272986457 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5357142857142857, + "precision": 0.4496353496353496, + "recall": 0.4523333809048095, + "f1": 0.4379382564939481 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9508196721311475, + "precision": 0.75, + "recall": 0.9741379310344828, + "f1": 0.8200589970501475 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.382227541927671, + "js_divergence": 0.09520406464421978 + } + } + }, + { + "fold": 2, + "best_params": { + "dropout": 0.4042053577703755, + "lr": 0.0004438352448528134, + "weight_decay": 0.09116356780872863, + "backbone_lr_ratio": 0.14631991134515426, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 17, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.3317227949570641, + "rmse": 0.5759538132151432, + "mae": 0.44960591762459706, + "r2": 0.3448534068400446 + }, + "delivery": { + "n_samples": 60, + "mse": 0.6524805809132939, + "rmse": 0.8077627008678315, + "mae": 0.5519438893922294, + "r2": 0.13143525632475672 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7023809523809523, + "precision": 0.6586452762923352, + "recall": 0.6957478005865103, + "f1": 0.662107803700724 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6904761904761905, + "precision": 0.640510613250929, + "recall": 0.7009803921568626, + "f1": 0.6591013004136229 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.3461542146527445, + "js_divergence": 0.09035975387137388 + } + } + }, + { + "fold": 3, + "best_params": { + "dropout": 0.12612237693012526, + "lr": 0.0009464809649969671, + "weight_decay": 0.07051438027639334, + "backbone_lr_ratio": 0.9343703462702573, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 16, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.630017453835127, + "rmse": 1.27672136891145, + "mae": 0.57301952361118, + "r2": 0.07804970427286173 + }, + "delivery": { + "n_samples": 59, + "mse": 0.7426488745509607, + "rmse": 0.8617707784271643, + "mae": 0.5635692574304797, + "r2": 0.27805705915089884 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7261904761904762, + "precision": 0.6489665003563792, + "recall": 0.6532258064516129, + "f1": 0.6509485094850949 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6190476190476191, + "precision": 0.5598290598290598, + "recall": 0.5861990950226245, + "f1": 0.5606349206349206 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9333333333333333, + "precision": 0.7142857142857143, + "recall": 0.9649122807017544, + "f1": 0.7818181818181817 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.32329647019984065, + "js_divergence": 0.07342759749750183 + } + } + }, + { + "fold": 4, + "best_params": { + "dropout": 0.20496607857127885, + "lr": 0.0009573095972231165, + "weight_decay": 0.00038069869878401156, + "backbone_lr_ratio": 0.04121720375781922, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 10, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.714982360155395, + "rmse": 0.845566295541275, + "mae": 0.5158508041640744, + "r2": 0.04326478992735272 + }, + "delivery": { + "n_samples": 58, + "mse": 0.9337723075176421, + "rmse": 0.966318947096476, + "mae": 0.615490146495145, + "r2": 0.0970575934388902 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7023809523809523, + "precision": 0.6423197492163009, + "recall": 0.6664222873900294, + "f1": 0.6480643539467068 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6428571428571429, + "precision": 0.5733695652173912, + "recall": 0.6085470085470085, + "f1": 0.5861678004535147 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.865909090909091 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.3041751519174415, + "js_divergence": 0.0828665483142382 + } + } + }, + { + "fold": 0, + "best_params": { + "dropout": 0.25888363054578495, + "lr": 0.0006536264082965735, + "weight_decay": 0.00932351457657158, + "backbone_lr_ratio": 0.1667894332719967, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 13, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.516128831525372, + "rmse": 1.2313118335845603, + "mae": 0.5887543337801134, + "r2": 0.07209183574368039 + }, + "delivery": { + "n_samples": 58, + "mse": 0.6296859698518799, + "rmse": 0.793527548262743, + "mae": 0.6343367575699913, + "r2": 0.199631134600178 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7976190476190477, + "precision": 0.7537037037037038, + "recall": 0.7929436920883821, + "f1": 0.765709598031173 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7023809523809523, + "precision": 0.6626672044094911, + "recall": 0.7147619047619048, + "f1": 0.6696677261461316 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9655172413793104, + "precision": 0.75, + "recall": 0.9821428571428572, + "f1": 0.8242424242424242 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.2648199540119534, + "js_divergence": 0.05676284260509181 + } + } + }, + { + "fold": 1, + "best_params": { + "dropout": 0.37015922545878527, + "lr": 0.0009352630241255291, + "weight_decay": 0.005516105285933501, + "backbone_lr_ratio": 0.12042458689005697, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 9, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.3644874868613839, + "rmse": 0.603727990788388, + "mae": 0.4844940258633523, + "r2": -0.11301520130806475 + }, + "delivery": { + "n_samples": 61, + "mse": 1.0796036294761662, + "rmse": 1.039039763183376, + "mae": 0.6630113559973533, + "r2": 0.1828178799589153 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6428571428571429, + "precision": 0.5498305084745763, + "recall": 0.5555555555555556, + "f1": 0.5509622238061297 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5238095238095238, + "precision": 0.4709239314502473, + "recall": 0.48534798534798534, + "f1": 0.4678875968992248 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9508196721311475, + "precision": 0.75, + "recall": 0.9741379310344828, + "f1": 0.8200589970501475 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.360009988398715, + "js_divergence": 0.08968488957961811 + } + } + }, + { + "fold": 2, + "best_params": { + "dropout": 0.4042053577703755, + "lr": 0.0004438352448528134, + "weight_decay": 0.09116356780872863, + "backbone_lr_ratio": 0.14631991134515426, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 17, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.3763557616627376, + "rmse": 0.613478411733239, + "mae": 0.48970496371787575, + "r2": 0.2567040950520848 + }, + "delivery": { + "n_samples": 60, + "mse": 0.703379623118135, + "rmse": 0.8386773057130704, + "mae": 0.5631674273948496, + "r2": 0.06367980912955817 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7142857142857143, + "precision": 0.6752941176470588, + "recall": 0.718475073313783, + "f1": 0.6785714285714286 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6904761904761905, + "precision": 0.6362280701754386, + "recall": 0.7009803921568626, + "f1": 0.6528571942316682 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.33457114910094327, + "js_divergence": 0.08453543607257773 + } + } + }, + { + "fold": 3, + "best_params": { + "dropout": 0.12612237693012526, + "lr": 0.0009464809649969671, + "weight_decay": 0.07051438027639334, + "backbone_lr_ratio": 0.9343703462702573, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 16, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.9634922624183433, + "rmse": 1.4012466815012778, + "mae": 0.6159229520596115, + "r2": -0.1105661891750771 + }, + "delivery": { + "n_samples": 59, + "mse": 0.8685113223217268, + "rmse": 0.9319395486412876, + "mae": 0.6189951948916256, + "r2": 0.1557038060862732 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7023809523809523, + "precision": 0.6503347534996957, + "recall": 0.6810850439882699, + "f1": 0.6554552912223134 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5833333333333334, + "precision": 0.5217673814165041, + "recall": 0.5463298139768727, + "f1": 0.5207287433261615 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9, + "precision": 0.6666666666666666, + "recall": 0.9473684210526316, + "f1": 0.7222222222222222 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.24978094522784985, + "js_divergence": 0.06060532361779249 + } + } + }, + { + "fold": 4, + "best_params": { + "dropout": 0.20496607857127885, + "lr": 0.0009573095972231165, + "weight_decay": 0.00038069869878401156, + "backbone_lr_ratio": 0.04121720375781922, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 10, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.666445125466178, + "rmse": 0.8163609039304723, + "mae": 0.49950868456203135, + "r2": 0.10821363903831627 + }, + "delivery": { + "n_samples": 58, + "mse": 0.9994847526266784, + "rmse": 0.9997423431198053, + "mae": 0.6419767608288033, + "r2": 0.033514743806195346 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7023809523809523, + "precision": 0.6503347534996957, + "recall": 0.6810850439882699, + "f1": 0.6554552912223134 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6547619047619048, + "precision": 0.5829995828118482, + "recall": 0.614957264957265, + "f1": 0.5934111227214677 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.865909090909091 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.2719915000176464, + "js_divergence": 0.0728753918681956 + } + } + }, + { + "fold": 0, + "best_params": { + "dropout": 0.25888363054578495, + "lr": 0.0006536264082965735, + "weight_decay": 0.00932351457657158, + "backbone_lr_ratio": 0.1667894332719967, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 13, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.515637174855938, + "rmse": 1.2311121698918983, + "mae": 0.5759322074836636, + "r2": 0.07239274172745469 + }, + "delivery": { + "n_samples": 58, + "mse": 0.6078859434204266, + "rmse": 0.7796704069159138, + "mae": 0.6284612801311344, + "r2": 0.22734028369354697 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.75, + "precision": 0.7147058823529412, + "recall": 0.7601568068424804, + "f1": 0.7211948790896159 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6904761904761905, + "precision": 0.6494435612082671, + "recall": 0.690952380952381, + "f1": 0.6540847855642071 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9655172413793104, + "precision": 0.75, + "recall": 0.9821428571428572, + "f1": 0.8242424242424242 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.28451853940500493, + "js_divergence": 0.06143098168643295 + } + } + }, + { + "fold": 1, + "best_params": { + "dropout": 0.37015922545878527, + "lr": 0.0009352630241255291, + "weight_decay": 0.005516105285933501, + "backbone_lr_ratio": 0.12042458689005697, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 9, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.36607084927706934, + "rmse": 0.6050378907779821, + "mae": 0.4742527622064309, + "r2": -0.11785022720432559 + }, + "delivery": { + "n_samples": 61, + "mse": 1.072597205810797, + "rmse": 1.0356626892047416, + "mae": 0.6899142471737549, + "r2": 0.18812123758800225 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6309523809523809, + "precision": 0.5535714285714286, + "recall": 0.5634920634920635, + "f1": 0.553421368547419 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5595238095238095, + "precision": 0.49943718228031947, + "recall": 0.516245659102802, + "f1": 0.5015974929297562 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9508196721311475, + "precision": 0.75, + "recall": 0.9741379310344828, + "f1": 0.8200589970501475 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.33867682492719764, + "js_divergence": 0.08667274774942817 + } + } + }, + { + "fold": 2, + "best_params": { + "dropout": 0.4042053577703755, + "lr": 0.0004438352448528134, + "weight_decay": 0.09116356780872863, + "backbone_lr_ratio": 0.14631991134515426, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 17, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.5053228762985142, + "rmse": 0.7108606588484935, + "mae": 0.544970038779346, + "r2": 0.001996347897613271 + }, + "delivery": { + "n_samples": 60, + "mse": 0.7184112972452166, + "rmse": 0.8475914683650471, + "mae": 0.5859764796448872, + "r2": 0.043670045518013456 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7261904761904762, + "precision": 0.6836007130124777, + "recall": 0.7265395894428153, + "f1": 0.6891391794046662 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6666666666666666, + "precision": 0.6104363472784525, + "recall": 0.6666666666666666, + "f1": 0.6224764228882279 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.34415499439775205, + "js_divergence": 0.08771435148935566 + } + } + }, + { + "fold": 3, + "best_params": { + "dropout": 0.12612237693012526, + "lr": 0.0009464809649969671, + "weight_decay": 0.07051438027639334, + "backbone_lr_ratio": 0.9343703462702573, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 16, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.7695798467912622, + "rmse": 1.3302555569480858, + "mae": 0.6023058171933854, + "r2": -0.0008878489144126256 + }, + "delivery": { + "n_samples": 59, + "mse": 0.7584981089453127, + "rmse": 0.8709179691252861, + "mae": 0.581826904741258, + "r2": 0.2626497202576923 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7261904761904762, + "precision": 0.6489665003563792, + "recall": 0.6532258064516129, + "f1": 0.6509485094850949 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6666666666666666, + "precision": 0.571467539552646, + "recall": 0.583107088989442, + "f1": 0.5753294324722896 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9333333333333333, + "precision": 0.7142857142857143, + "recall": 0.9649122807017544, + "f1": 0.7818181818181817 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.26330683934899884, + "js_divergence": 0.05955508350882244 + } + } + }, + { + "fold": 4, + "best_params": { + "dropout": 0.20496607857127885, + "lr": 0.0009573095972231165, + "weight_decay": 0.00038069869878401156, + "backbone_lr_ratio": 0.04121720375781922, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 10, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.6649265530357125, + "rmse": 0.8154302870483242, + "mae": 0.5136514454137623, + "r2": 0.1102456775811359 + }, + "delivery": { + "n_samples": 58, + "mse": 0.9676062147836739, + "rmse": 0.9836697691723956, + "mae": 0.6175258674755179, + "r2": 0.06434076364622787 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7738095238095238, + "precision": 0.7122033898305085, + "recall": 0.7294721407624634, + "f1": 0.719359943731317 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6428571428571429, + "precision": 0.5648809523809524, + "recall": 0.5871794871794872, + "f1": 0.573676603432701 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.865909090909091 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.29835153058548175, + "js_divergence": 0.08270536087983824 + } + } + } + ], + "summary_stats": { + "size": { + "mse_mean": 0.9543035685322051, + "mse_std": 0.5981213334036609, + "rmse_mean": 0.9270408839286446, + "rmse_std": 0.3080564364803995, + "mae_mean": 0.5320147123966243, + "mae_std": 0.05351966804020229, + "r2_mean": 0.04726470045023875, + "r2_std": 0.126102268122003 + }, + "delivery": { + "mse_mean": 0.8270684480229857, + "mse_std": 0.17693567699093907, + "rmse_mean": 0.9042243053458754, + "rmse_std": 0.09719492602371275, + "mae_mean": 0.615708873112371, + "mae_std": 0.039477393320796444, + "r2_mean": 0.1574941704451898, + "r2_std": 0.07929093899582297 + }, + "pdi": { + "accuracy_mean": 0.7142857142857143, + "accuracy_std": 0.04123930494211614, + "precision_mean": 0.657051911797391, + "precision_std": 0.05438419992076548, + "recall_mean": 0.6835853401829536, + "recall_std": 0.06618289811754123, + "f1_mean": 0.6610780721243127, + "f1_std": 0.05631636693524698 + }, + "ee": { + "accuracy_mean": 0.6365079365079365, + "accuracy_std": 0.05715387900758358, + "precision_mean": 0.5754990153190521, + "precision_std": 0.06434539420274109, + "recall_mean": 0.6092582823339125, + "recall_std": 0.0807804696823919, + "f1_mean": 0.5812582234730809, + "f1_std": 0.0702133273457957 + }, + "toxic": { + "accuracy_mean": 0.9478174120312263, + "accuracy_std": 0.018643838747495024, + "precision_mean": 0.7425396825396825, + "precision_std": 0.03650310527039388, + "recall_mean": 0.9726010428369776, + "recall_std": 0.009861789747008767, + "f1_mean": 0.8108612462594762, + "f1_std": 0.038209835226095154 + }, + "biodist": { + "kl_divergence_mean": 0.3107850118699246, + "kl_divergence_std": 0.03838394769553977, + "js_divergence_mean": 0.07653345565385793, + "js_divergence_std": 0.012738301434340309 + } + } +} \ No newline at end of file diff --git a/results/pretrained_encoders/moe_pretrain_s42_summary.json b/results/pretrained_encoders/moe_pretrain_s42_summary.json new file mode 100644 index 0000000..3dbeb97 --- /dev/null +++ b/results/pretrained_encoders/moe_pretrain_s42_summary.json @@ -0,0 +1,977 @@ +{ + "fold_results": [ + { + "fold": 0, + "best_params": { + "dropout": 0.4884866103742062, + "lr": 0.0002617457281978971, + "weight_decay": 9.524902838811199e-05, + "backbone_lr_ratio": 0.9539957173265954, + "moe_n_experts": 4, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 23, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.5279509180840005, + "rmse": 1.2361031179007682, + "mae": 0.578313840710255, + "r2": 0.0648564277702961 + }, + "delivery": { + "n_samples": 58, + "mse": 0.6655882063296256, + "rmse": 0.8158358942395374, + "mae": 0.674296624894286, + "r2": 0.15399722555537465 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7619047619047619, + "precision": 0.7122257053291536, + "recall": 0.7412687099073414, + "f1": 0.7214854111405835 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7142857142857143, + "precision": 0.6748583260680036, + "recall": 0.7142857142857144, + "f1": 0.6773495605138607 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9655172413793104, + "precision": 0.75, + "recall": 0.9821428571428572, + "f1": 0.8242424242424242 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.2500928345559933, + "js_divergence": 0.05156067014803717 + } + } + }, + { + "fold": 1, + "best_params": { + "dropout": 0.10335173439505978, + "lr": 0.000196757629557381, + "weight_decay": 0.0003725444839719432, + "backbone_lr_ratio": 0.05035847475715459, + "moe_n_experts": 4, + "moe_top_k": 1, + "moe_expert_hidden_mult": 1, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 19, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.34213962908738765, + "rmse": 0.584927028856923, + "mae": 0.45376784687063526, + "r2": -0.044772788836473065 + }, + "delivery": { + "n_samples": 61, + "mse": 1.129722640308642, + "rmse": 1.0628841142423016, + "mae": 0.6647483138825561, + "r2": 0.1448814017846829 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6309523809523809, + "precision": 0.5535714285714286, + "recall": 0.5634920634920635, + "f1": 0.553421368547419 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5238095238095238, + "precision": 0.46498599439775906, + "recall": 0.46436896436896435, + "f1": 0.4483749534478767 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9508196721311475, + "precision": 0.75, + "recall": 0.9741379310344828, + "f1": 0.8200589970501475 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.49848379935596204, + "js_divergence": 0.1305599268889464 + } + } + }, + { + "fold": 2, + "best_params": { + "dropout": 0.2514825943283291, + "lr": 0.00029260650717186565, + "weight_decay": 0.0001495891705646636, + "backbone_lr_ratio": 0.2434609849039314, + "moe_n_experts": 2, + "moe_top_k": 2, + "moe_expert_hidden_mult": 1, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 20, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.3851340701995523, + "rmse": 0.6205917097412375, + "mae": 0.4943395415542736, + "r2": 0.2393670925336251 + }, + "delivery": { + "n_samples": 60, + "mse": 0.7088597182801071, + "rmse": 0.8419380727108776, + "mae": 0.5629216438857838, + "r2": 0.0563848526375017 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6904761904761905, + "precision": 0.6505882352941177, + "recall": 0.6876832844574781, + "f1": 0.6517857142857143 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6666666666666666, + "precision": 0.6139316239316239, + "recall": 0.6785714285714285, + "f1": 0.6277576489533011 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.2665180519269797, + "js_divergence": 0.06450481128744694 + } + } + }, + { + "fold": 3, + "best_params": { + "dropout": 0.40463841601213524, + "lr": 0.0002572978619380859, + "weight_decay": 2.0354315009822035e-05, + "backbone_lr_ratio": 0.8823918523568081, + "moe_n_experts": 4, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 19, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.820237956048593, + "rmse": 1.3491619458199202, + "mae": 0.6053206579400653, + "r2": -0.029540461621649028 + }, + "delivery": { + "n_samples": 59, + "mse": 0.6693395316760985, + "rmse": 0.8181317324710603, + "mae": 0.5448598891843931, + "r2": 0.34932245037470455 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6159874608150471, + "recall": 0.6356304985337243, + "f1": 0.6199095022624435 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6309523809523809, + "precision": 0.5653410228683567, + "recall": 0.5826043237807944, + "f1": 0.5650366426763943 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9333333333333333, + "precision": 0.7142857142857143, + "recall": 0.9649122807017544, + "f1": 0.7818181818181817 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.29215591110500194, + "js_divergence": 0.07269076599614506 + } + } + }, + { + "fold": 4, + "best_params": { + "dropout": 0.3730299609537786, + "lr": 0.0007646220605180808, + "weight_decay": 0.004588258639584098, + "backbone_lr_ratio": 0.17129190112240483, + "moe_n_experts": 2, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 15, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.7676733172863143, + "rmse": 0.8761696852130381, + "mae": 0.5270063721386361, + "r2": -0.02724225576902284 + }, + "delivery": { + "n_samples": 58, + "mse": 0.9040350302083842, + "rmse": 0.9508075673912068, + "mae": 0.6287906430019387, + "r2": 0.12581304969093698 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7261904761904762, + "precision": 0.6836007130124777, + "recall": 0.7265395894428153, + "f1": 0.6891391794046662 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7023809523809523, + "precision": 0.6645240097721175, + "recall": 0.7038461538461539, + "f1": 0.6728729539789446 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.865909090909091 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.20431460217894573, + "js_divergence": 0.05043386132176378 + } + } + }, + { + "fold": 0, + "best_params": { + "dropout": 0.4884866103742062, + "lr": 0.0002617457281978971, + "weight_decay": 9.524902838811199e-05, + "backbone_lr_ratio": 0.9539957173265954, + "moe_n_experts": 4, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 23, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.5550563846692986, + "rmse": 1.2470189993217018, + "mae": 0.5954494658094572, + "r2": 0.04826721502168674 + }, + "delivery": { + "n_samples": 58, + "mse": 0.704182837608261, + "rmse": 0.8391560269748772, + "mae": 0.6780449980548744, + "r2": 0.10494112024899149 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7142857142857143, + "precision": 0.6737967914438503, + "recall": 0.7084818246614397, + "f1": 0.6785714285714286 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7261904761904762, + "precision": 0.6931286549707601, + "recall": 0.7380952380952381, + "f1": 0.7007070707070707 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9655172413793104, + "precision": 0.75, + "recall": 0.9821428571428572, + "f1": 0.8242424242424242 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.2453488369022721, + "js_divergence": 0.052406074597242516 + } + } + }, + { + "fold": 1, + "best_params": { + "dropout": 0.10335173439505978, + "lr": 0.000196757629557381, + "weight_decay": 0.0003725444839719432, + "backbone_lr_ratio": 0.05035847475715459, + "moe_n_experts": 4, + "moe_top_k": 1, + "moe_expert_hidden_mult": 1, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 19, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.3727706424788934, + "rmse": 0.610549459486202, + "mae": 0.48057969708350445, + "r2": -0.13830901371429016 + }, + "delivery": { + "n_samples": 61, + "mse": 1.0803452340552668, + "rmse": 1.0393965720817377, + "mae": 0.6485996645432516, + "r2": 0.18225653875401804 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6309523809523809, + "precision": 0.5535714285714286, + "recall": 0.5634920634920635, + "f1": 0.553421368547419 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5476190476190477, + "precision": 0.49737237237237236, + "recall": 0.5073022215879358, + "f1": 0.4920019687461548 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9508196721311475, + "precision": 0.75, + "recall": 0.9741379310344828, + "f1": 0.8200589970501475 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.5055220562600806, + "js_divergence": 0.1300376337593102 + } + } + }, + { + "fold": 2, + "best_params": { + "dropout": 0.2514825943283291, + "lr": 0.00029260650717186565, + "weight_decay": 0.0001495891705646636, + "backbone_lr_ratio": 0.2434609849039314, + "moe_n_experts": 2, + "moe_top_k": 2, + "moe_expert_hidden_mult": 1, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 20, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.501474748290053, + "rmse": 0.708148817897801, + "mae": 0.5538181106981245, + "r2": 0.009596331959947801 + }, + "delivery": { + "n_samples": 60, + "mse": 0.739867919930154, + "rmse": 0.8601557532971305, + "mae": 0.5739390922787909, + "r2": 0.015107561778814449 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7380952380952381, + "precision": 0.6923076923076923, + "recall": 0.7346041055718475, + "f1": 0.6998050682261208 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6228947368421053, + "recall": 0.6851073762838468, + "f1": 0.63836444060848 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.2914659417557568, + "js_divergence": 0.07149076417555128 + } + } + }, + { + "fold": 3, + "best_params": { + "dropout": 0.40463841601213524, + "lr": 0.0002572978619380859, + "weight_decay": 2.0354315009822035e-05, + "backbone_lr_ratio": 0.8823918523568081, + "moe_n_experts": 4, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 19, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.7508482794444888, + "rmse": 1.3231962361813492, + "mae": 0.5995979247704507, + "r2": 0.009706868347147068 + }, + "delivery": { + "n_samples": 59, + "mse": 0.6800398997076931, + "rmse": 0.824645317520019, + "mae": 0.5383539610084588, + "r2": 0.33892042132757505 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7023809523809523, + "precision": 0.6423197492163009, + "recall": 0.6664222873900294, + "f1": 0.6480643539467068 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6666666666666666, + "precision": 0.5898730898730898, + "recall": 0.6022121669180492, + "f1": 0.5907255864702673 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9166666666666666, + "precision": 0.6875, + "recall": 0.956140350877193, + "f1": 0.749791492910759 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.22697074487882685, + "js_divergence": 0.05572372821276913 + } + } + }, + { + "fold": 4, + "best_params": { + "dropout": 0.3730299609537786, + "lr": 0.0007646220605180808, + "weight_decay": 0.004588258639584098, + "backbone_lr_ratio": 0.17129190112240483, + "moe_n_experts": 2, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 15, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.7388933186856751, + "rmse": 0.8595890405802502, + "mae": 0.4949821270059883, + "r2": 0.01126895729235744 + }, + "delivery": { + "n_samples": 58, + "mse": 1.0735857604041814, + "rmse": 1.0361398363175607, + "mae": 0.6398222603160759, + "r2": -0.038139707447659266 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7023809523809523, + "precision": 0.6345029239766082, + "recall": 0.6517595307917888, + "f1": 0.6398559423769508 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7261904761904762, + "precision": 0.662962962962963, + "recall": 0.7064102564102565, + "f1": 0.6789544933874829 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.865909090909091 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.2849032950499658, + "js_divergence": 0.0742269697587313 + } + } + }, + { + "fold": 0, + "best_params": { + "dropout": 0.4884866103742062, + "lr": 0.0002617457281978971, + "weight_decay": 9.524902838811199e-05, + "backbone_lr_ratio": 0.9539957173265954, + "moe_n_experts": 4, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 23, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.575393168842647, + "rmse": 1.255146672242988, + "mae": 0.5922032207340361, + "r2": 0.035820602519645584 + }, + "delivery": { + "n_samples": 58, + "mse": 0.6805529919450708, + "rmse": 0.8249563576002495, + "mae": 0.6910147559026192, + "r2": 0.13497608000435835 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7976190476190477, + "precision": 0.75, + "recall": 0.7794012829650748, + "f1": 0.7606837606837606 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7380952380952381, + "precision": 0.7064443475733798, + "recall": 0.7547619047619047, + "f1": 0.7126436781609194 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9655172413793104, + "precision": 0.75, + "recall": 0.9821428571428572, + "f1": 0.8242424242424242 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.23590056396282402, + "js_divergence": 0.047894235451583785 + } + } + }, + { + "fold": 1, + "best_params": { + "dropout": 0.10335173439505978, + "lr": 0.000196757629557381, + "weight_decay": 0.0003725444839719432, + "backbone_lr_ratio": 0.05035847475715459, + "moe_n_experts": 4, + "moe_top_k": 1, + "moe_expert_hidden_mult": 1, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 19, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.3579244858231685, + "rmse": 0.5982679047242703, + "mae": 0.46379138462777647, + "r2": -0.09297412943304506 + }, + "delivery": { + "n_samples": 61, + "mse": 1.1096558552074713, + "rmse": 1.0534020387332992, + "mae": 0.6546663998885721, + "r2": 0.1600705115131673 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6547619047619048, + "precision": 0.5696286472148542, + "recall": 0.5793650793650793, + "f1": 0.571654650958326 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5476190476190477, + "precision": 0.49253034547152197, + "recall": 0.49681271109842545, + "f1": 0.4801587301587302 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.502867985249319, + "js_divergence": 0.13123665503825419 + } + } + }, + { + "fold": 2, + "best_params": { + "dropout": 0.2514825943283291, + "lr": 0.00029260650717186565, + "weight_decay": 0.0001495891705646636, + "backbone_lr_ratio": 0.2434609849039314, + "moe_n_experts": 2, + "moe_top_k": 2, + "moe_expert_hidden_mult": 1, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 20, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.618457534400238, + "rmse": 0.7864207108159335, + "mae": 0.5679304331563235, + "r2": -0.22144258048008303 + }, + "delivery": { + "n_samples": 60, + "mse": 0.7362245815365097, + "rmse": 0.8580353031994137, + "mae": 0.5702473803190514, + "r2": 0.019957476658378215 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6523864289821737, + "recall": 0.6942815249266863, + "f1": 0.6473332296687918 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6228947368421053, + "recall": 0.6851073762838468, + "f1": 0.63836444060848 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.26356343521196823, + "js_divergence": 0.06378606399643424 + } + } + }, + { + "fold": 3, + "best_params": { + "dropout": 0.40463841601213524, + "lr": 0.0002572978619380859, + "weight_decay": 2.0354315009822035e-05, + "backbone_lr_ratio": 0.8823918523568081, + "moe_n_experts": 4, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 19, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.8408361971730496, + "rmse": 1.3567741879815702, + "mae": 0.6257544057668153, + "r2": -0.04119098379947639 + }, + "delivery": { + "n_samples": 59, + "mse": 0.737776343129472, + "rmse": 0.8589390799873249, + "mae": 0.5497218715026975, + "r2": 0.28279373860245827 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6904761904761905, + "precision": 0.6333333333333333, + "recall": 0.6583577712609971, + "f1": 0.6379310344827587 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6428571428571429, + "precision": 0.5640697674418605, + "recall": 0.5891402714932127, + "f1": 0.5710279611967066 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9333333333333333, + "precision": 0.7142857142857143, + "recall": 0.9649122807017544, + "f1": 0.7818181818181817 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.24488500401793606, + "js_divergence": 0.060842241588791174 + } + } + }, + { + "fold": 4, + "best_params": { + "dropout": 0.3730299609537786, + "lr": 0.0007646220605180808, + "weight_decay": 0.004588258639584098, + "backbone_lr_ratio": 0.17129190112240483, + "moe_n_experts": 2, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 15, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.712297885673984, + "rmse": 0.8439774201209319, + "mae": 0.5084991393599748, + "r2": 0.046856950237922024 + }, + "delivery": { + "n_samples": 58, + "mse": 1.1055967310671657, + "rmse": 1.0514735997956228, + "mae": 0.6599975297677105, + "r2": -0.06909378763839769 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7619047619047619, + "precision": 0.711111111111111, + "recall": 0.750733137829912, + "f1": 0.7214854111405835 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7023809523809523, + "precision": 0.6536363636363637, + "recall": 0.7038461538461539, + "f1": 0.6717592592592593 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.865909090909091 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.1782104997564988, + "js_divergence": 0.04297224929274232 + } + } + } + ], + "summary_stats": { + "size": { + "mse_mean": 0.9911392357458229, + "mse_std": 0.5817120308358389, + "rmse_mean": 0.9504028624589923, + "rmse_std": 0.29643487442535593, + "mae_mean": 0.542756944548421, + "mae_std": 0.054981488862431024, + "r2_mean": -0.008648784531427449, + "r2_std": 0.09954199193808574 + }, + "delivery": { + "mse_mean": 0.8483582187596068, + "mse_std": 0.18619352158793398, + "rmse_mean": 0.9157264844374813, + "rmse_std": 0.09901123400593626, + "mae_mean": 0.6186683352287373, + "mae_std": 0.05324906806818642, + "r2_mean": 0.13081259558966032, + "r2_std": 0.1207826593867956 + }, + "pdi": { + "accuracy_mean": 0.703968253968254, + "accuracy_std": 0.04618202071738462, + "precision_mean": 0.6485954432786384, + "precision_std": 0.056485137303304574, + "recall_mean": 0.6761008502725561, + "recall_std": 0.06599285309996766, + "f1_mean": 0.6529698282829115, + "f1_std": 0.05951390665637187 + }, + "ee": { + "accuracy_mean": 0.6595238095238095, + "accuracy_std": 0.0668930397006523, + "precision_mean": 0.6059632236682923, + "precision_std": 0.07313293968503662, + "recall_mean": 0.640831484108795, + "recall_std": 0.0904387508717925, + "f1_mean": 0.6110732925915952, + "f1_std": 0.08105368908168387 + }, + "toxic": { + "accuracy_mean": 0.947835626967474, + "accuracy_std": 0.01646827276703399, + "precision_mean": 0.741547619047619, + "precision_std": 0.034629149636327315, + "recall_mean": 0.9726111255149369, + "recall_std": 0.008721274087923442, + "f1_mean": 0.810171454978226, + "f1_std": 0.03512408564460891 + }, + "biodist": { + "kl_divergence_mean": 0.299413570811222, + "kl_divergence_std": 0.1056856517385418, + "js_divergence_mean": 0.0733577767675833, + "js_divergence_std": 0.029986885602095156 + } + } +} \ No newline at end of file diff --git a/results/pretrained_encoders/molformer_pretrain_s42_summary.json b/results/pretrained_encoders/molformer_pretrain_s42_summary.json new file mode 100644 index 0000000..74d9b9d --- /dev/null +++ b/results/pretrained_encoders/molformer_pretrain_s42_summary.json @@ -0,0 +1,932 @@ +{ + "fold_results": [ + { + "fold": 0, + "best_params": { + "dropout": 0.3980773018180746, + "lr": 0.0009896030589860336, + "weight_decay": 0.0002988554096805342, + "backbone_lr_ratio": 0.2882797738700643, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 13, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.6246690728469169, + "rmse": 1.274625071480597, + "mae": 0.5944420359916536, + "r2": 0.005662536347478486 + }, + "delivery": { + "n_samples": 58, + "mse": 0.7102819707127421, + "rmse": 0.842782279543621, + "mae": 0.6625874312594533, + "r2": 0.09718875402761229 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.8333333333333334, + "precision": 0.7891525423728813, + "recall": 0.8039914468995011, + "f1": 0.7958333333333334 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7142857142857143, + "precision": 0.6829362569400521, + "recall": 0.7314285714285714, + "f1": 0.687769981711114 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9655172413793104, + "precision": 0.75, + "recall": 0.9821428571428572, + "f1": 0.8242424242424242 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.32097759110945434, + "js_divergence": 0.06712981080576393 + } + } + }, + { + "fold": 1, + "best_params": { + "dropout": 0.396946460618591, + "lr": 0.0004854064403598687, + "weight_decay": 0.00010992238518795333, + "backbone_lr_ratio": 0.23864875345272699, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 10, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.38773907280523073, + "rmse": 0.6226869781882634, + "mae": 0.478045186826161, + "r2": -0.18401727831452308 + }, + "delivery": { + "n_samples": 61, + "mse": 1.1492300175260897, + "rmse": 1.0720214631834988, + "mae": 0.7018961062746458, + "r2": 0.13011572349706002 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.5952380952380952, + "precision": 0.5664174813110983, + "recall": 0.5873015873015872, + "f1": 0.5523510971786834 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5476190476190477, + "precision": 0.4826358616681197, + "recall": 0.4884639170353456, + "f1": 0.47545555874980455 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.4828981973271936, + "js_divergence": 0.12323903696473625 + } + } + }, + { + "fold": 2, + "best_params": { + "dropout": 0.10827786083228613, + "lr": 0.0004876914007718381, + "weight_decay": 0.0852885971769344, + "backbone_lr_ratio": 0.21714091995810716, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 8, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.4123269005771989, + "rmse": 0.6421268570751413, + "mae": 0.49745388741471935, + "r2": 0.1856617383911774 + }, + "delivery": { + "n_samples": 60, + "mse": 0.6503294595263629, + "rmse": 0.8064300710702466, + "mae": 0.5487457474150385, + "r2": 0.1342987717315124 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.5595238095238095, + "precision": 0.5652297220646625, + "recall": 0.5843108504398826, + "f1": 0.5357729648991785 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6547619047619048, + "precision": 0.6053846153846153, + "recall": 0.6720354808590102, + "f1": 0.6203502415458937 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.3376426852856137, + "js_divergence": 0.09061480035897176 + } + } + }, + { + "fold": 3, + "best_params": { + "dropout": 0.1463802845991778, + "lr": 0.00017979738268552602, + "weight_decay": 0.026590603512062855, + "backbone_lr_ratio": 0.19962036354162335, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 18, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.7510092489615507, + "rmse": 1.3232570608016987, + "mae": 0.6278676227834749, + "r2": 0.009615822761402493 + }, + "delivery": { + "n_samples": 59, + "mse": 0.7222390124885935, + "rmse": 0.84984646406783, + "mae": 0.57879668119032, + "r2": 0.29789786998972123 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7380952380952381, + "precision": 0.6785714285714286, + "recall": 0.7052785923753666, + "f1": 0.6867796610169492 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5476190476190477, + "precision": 0.5483829895594602, + "recall": 0.575062845651081, + "f1": 0.5199383832295225 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9333333333333333, + "precision": 0.7142857142857143, + "recall": 0.9649122807017544, + "f1": 0.7818181818181817 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.4613132992323466, + "js_divergence": 0.12165500439647682 + } + } + }, + { + "fold": 4, + "best_params": { + "dropout": 0.35317424690388177, + "lr": 0.0009765192272158632, + "weight_decay": 0.00012175015526770499, + "backbone_lr_ratio": 0.023808704109897987, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 18, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.6604880002050739, + "rmse": 0.8127041283302762, + "mae": 0.4948491998892721, + "r2": 0.11618501260741076 + }, + "delivery": { + "n_samples": 58, + "mse": 0.936642398150463, + "rmse": 0.9678028715345202, + "mae": 0.6353725126848139, + "r2": 0.09428226317670119 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6071428571428571, + "precision": 0.6374353076480737, + "recall": 0.6752199413489737, + "f1": 0.5942028985507246 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6904761904761905, + "precision": 0.6475761475761476, + "recall": 0.7401709401709402, + "f1": 0.6653096672637463 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.865909090909091 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.40727903738537147, + "js_divergence": 0.10249950448751813 + } + } + }, + { + "fold": 0, + "best_params": { + "dropout": 0.3980773018180746, + "lr": 0.0009896030589860336, + "weight_decay": 0.0002988554096805342, + "backbone_lr_ratio": 0.2882797738700643, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 13, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.545312966192772, + "rmse": 1.2431061765564404, + "mae": 0.5886967062770602, + "r2": 0.054230426962613154 + }, + "delivery": { + "n_samples": 58, + "mse": 0.6668586486255093, + "rmse": 0.8166141369248449, + "mae": 0.6276618008790859, + "r2": 0.15238241673384167 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.8452380952380952, + "precision": 0.8041666666666667, + "recall": 0.8121881682109764, + "f1": 0.8079831193951117 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7261904761904762, + "precision": 0.6914328121224673, + "recall": 0.7452380952380953, + "f1": 0.701447533036203 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9827586206896551, + "precision": 0.9912280701754386, + "recall": 0.75, + "f1": 0.8289085545722714 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.27259534788302564, + "js_divergence": 0.06020453275369037 + } + } + }, + { + "fold": 1, + "best_params": { + "dropout": 0.396946460618591, + "lr": 0.0004854064403598687, + "weight_decay": 0.00010992238518795333, + "backbone_lr_ratio": 0.23864875345272699, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 10, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.4005803081504934, + "rmse": 0.6329141396354591, + "mae": 0.4951437964946741, + "r2": -0.2232298457086057 + }, + "delivery": { + "n_samples": 61, + "mse": 1.1667453321847503, + "rmse": 1.0801598641797197, + "mae": 0.6814669590809794, + "r2": 0.11685789296077709 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.5595238095238095, + "precision": 0.5477272727272727, + "recall": 0.5634920634920635, + "f1": 0.5238241152137275 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5238095238095238, + "precision": 0.456992337164751, + "recall": 0.46650968079539507, + "f1": 0.45625740933077347 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.5730307313158203, + "js_divergence": 0.14005679605749516 + } + } + }, + { + "fold": 2, + "best_params": { + "dropout": 0.10827786083228613, + "lr": 0.0004876914007718381, + "weight_decay": 0.0852885971769344, + "backbone_lr_ratio": 0.21714091995810716, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 8, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.45499801818836005, + "rmse": 0.6745354091434785, + "mae": 0.5309686183796397, + "r2": 0.1013870435125862 + }, + "delivery": { + "n_samples": 60, + "mse": 0.6030715996767626, + "rmse": 0.7765768472448574, + "mae": 0.5282929584461574, + "r2": 0.1972071741079553 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6336898395721925, + "recall": 0.6649560117302054, + "f1": 0.635076427996782 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6547619047619048, + "precision": 0.5992402659069326, + "recall": 0.6507936507936507, + "f1": 0.6101851851851852 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.358983754945942, + "js_divergence": 0.09298507446003766 + } + } + }, + { + "fold": 3, + "best_params": { + "dropout": 0.1463802845991778, + "lr": 0.00017979738268552602, + "weight_decay": 0.026590603512062855, + "backbone_lr_ratio": 0.19962036354162335, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 18, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.6469751588239248, + "rmse": 1.2833452999188975, + "mae": 0.6211532356262387, + "r2": 0.06845829708118345 + }, + "delivery": { + "n_samples": 59, + "mse": 0.7509799772948998, + "rmse": 0.866591009239595, + "mae": 0.6103691533050042, + "r2": 0.2699582374579258 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6428571428571429, + "precision": 0.6545454545454545, + "recall": 0.6994134897360704, + "f1": 0.625668449197861 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6190476190476191, + "precision": 0.5681642512077295, + "recall": 0.6041478129713425, + "f1": 0.563984674329502 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9333333333333333, + "precision": 0.7142857142857143, + "recall": 0.9649122807017544, + "f1": 0.7818181818181817 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.4523145833037008, + "js_divergence": 0.1183277351587518 + } + } + }, + { + "fold": 4, + "best_params": { + "dropout": 0.35317424690388177, + "lr": 0.0009765192272158632, + "weight_decay": 0.00012175015526770499, + "backbone_lr_ratio": 0.023808704109897987, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 18, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.6556117434601818, + "rmse": 0.8096985509806608, + "mae": 0.5088372581716006, + "r2": 0.12271004984074696 + }, + "delivery": { + "n_samples": 58, + "mse": 1.0003346592105318, + "rmse": 1.0001673156080095, + "mae": 0.6612379994114925, + "r2": 0.03269289816995369 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7142857142857143, + "precision": 0.6592592592592592, + "recall": 0.6891495601173021, + "f1": 0.6657824933687002 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6428571428571429, + "precision": 0.6072310405643738, + "recall": 0.6940170940170941, + "f1": 0.6169253712734788 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.865909090909091 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.4083994319724002, + "js_divergence": 0.1006941777950375 + } + } + }, + { + "fold": 0, + "best_params": { + "dropout": 0.3980773018180746, + "lr": 0.0009896030589860336, + "weight_decay": 0.0002988554096805342, + "backbone_lr_ratio": 0.2882797738700643, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 13, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.444963857475926, + "rmse": 1.2020664946149717, + "mae": 0.575902431103659, + "r2": 0.11564655158081194 + }, + "delivery": { + "n_samples": 58, + "mse": 0.6487752159789035, + "rmse": 0.8054658378720376, + "mae": 0.6530212075288954, + "r2": 0.1753675508528415 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.8333333333333334, + "precision": 0.7936507936507937, + "recall": 0.7769066286528867, + "f1": 0.7844574780058651 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7261904761904762, + "precision": 0.6868692000270947, + "recall": 0.7452380952380953, + "f1": 0.6987547892720306 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9482758620689655, + "precision": 0.7, + "recall": 0.9732142857142857, + "f1": 0.7719528178243775 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.32634100712155006, + "js_divergence": 0.06706463335858984 + } + } + }, + { + "fold": 1, + "best_params": { + "dropout": 0.396946460618591, + "lr": 0.0004854064403598687, + "weight_decay": 0.00010992238518795333, + "backbone_lr_ratio": 0.23864875345272699, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 10, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.45034502529074416, + "rmse": 0.6710775106429541, + "mae": 0.5225315168943434, + "r2": -0.37519359936953656 + }, + "delivery": { + "n_samples": 61, + "mse": 1.1274144896048421, + "rmse": 1.0617977630438116, + "mae": 0.6428290372561725, + "r2": 0.14662850547534068 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6904761904761905, + "precision": 0.5972915181753385, + "recall": 0.6031746031746031, + "f1": 0.5997067448680352 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5238095238095238, + "precision": 0.4141642671054436, + "recall": 0.42669235526378385, + "f1": 0.40981240981240985 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9016393442622951, + "precision": 0.6666666666666666, + "recall": 0.9482758620689655, + "f1": 0.7227272727272727 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.5404966442274723, + "js_divergence": 0.13248730719985696 + } + } + }, + { + "fold": 2, + "best_params": { + "dropout": 0.10827786083228613, + "lr": 0.0004876914007718381, + "weight_decay": 0.0852885971769344, + "backbone_lr_ratio": 0.21714091995810716, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 8, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.5404522607814335, + "rmse": 0.7351545829153441, + "mae": 0.5627435291618375, + "r2": -0.06738355880059732 + }, + "delivery": { + "n_samples": 60, + "mse": 0.6347980231433975, + "rmse": 0.7967421308951834, + "mae": 0.5446047547040507, + "r2": 0.1549738055263823 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.5952380952380952, + "precision": 0.6068181818181818, + "recall": 0.6378299120234604, + "f1": 0.5757575757575757 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6190476190476191, + "precision": 0.5553467000835423, + "recall": 0.588702147525677, + "f1": 0.5573950928686081 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9344262295081968, + "precision": 0.7142857142857143, + "recall": 0.9655172413793103, + "f1": 0.7821428571428571 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.34103334231152865, + "js_divergence": 0.0899550896776649 + } + } + }, + { + "fold": 3, + "best_params": { + "dropout": 0.1463802845991778, + "lr": 0.00017979738268552602, + "weight_decay": 0.026590603512062855, + "backbone_lr_ratio": 0.19962036354162335, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 18, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.633019754709887, + "rmse": 1.2778966134667886, + "mae": 0.6318143501271865, + "r2": 0.07635158001484588 + }, + "delivery": { + "n_samples": 59, + "mse": 0.778873610785293, + "rmse": 0.8825381639256702, + "mae": 0.591008120058578, + "r2": 0.2428423116373991 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.5983050847457627, + "recall": 0.6063049853372434, + "f1": 0.6011957095129242 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6428571428571429, + "precision": 0.5736842105263158, + "recall": 0.6070889894419307, + "f1": 0.5785510785510786 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9333333333333333, + "precision": 0.7142857142857143, + "recall": 0.9649122807017544, + "f1": 0.7818181818181817 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.46990571185549446, + "js_divergence": 0.12454201663905487 + } + } + }, + { + "fold": 4, + "best_params": { + "dropout": 0.35317424690388177, + "lr": 0.0009765192272158632, + "weight_decay": 0.00012175015526770499, + "backbone_lr_ratio": 0.023808704109897987, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 18, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.7197383052375195, + "rmse": 0.8483739182916454, + "mae": 0.5446056428176927, + "r2": 0.0369007446434223 + }, + "delivery": { + "n_samples": 58, + "mse": 1.0250908773516882, + "rmse": 1.0124677166960379, + "mae": 0.6531032593599682, + "r2": 0.008754043905628395 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6190476190476191, + "precision": 0.6079545454545454, + "recall": 0.6392961876832844, + "f1": 0.5909920876445527 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6275303643724697, + "recall": 0.7021367521367522, + "f1": 0.6415056360708534 + }, + "toxic": { + "n_samples": 59, + "accuracy": 0.9661016949152542, + "precision": 0.8, + "recall": 0.9821428571428572, + "f1": 0.865909090909091 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.40077254800841094, + "js_divergence": 0.10315238600509395 + } + } + } + ], + "summary_stats": { + "size": { + "mse_mean": 0.9552153129138143, + "mse_std": 0.544124280885305, + "rmse_mean": 0.9369045861361744, + "rmse_std": 0.27825367812630597, + "mae_mean": 0.5516703345306142, + "mae_std": 0.050933508755883206, + "r2_mean": 0.0028657014366944233, + "r2_std": 0.1484374631238745 + }, + "delivery": { + "mse_mean": 0.8381110194840553, + "mse_std": 0.19956001015608432, + "rmse_mean": 0.9092002623352989, + "rmse_std": 0.1070789542976535, + "mae_mean": 0.6213995819236436, + "mae_std": 0.05084304982304164, + "r2_mean": 0.15009654795004349, + "r2_std": 0.07708969134269554 + }, + "pdi": { + "accuracy_mean": 0.6793650793650793, + "accuracy_std": 0.09408695848584789, + "precision_mean": 0.6493476732389074, + "precision_std": 0.08116705378595848, + "recall_mean": 0.6699209352348937, + "recall_std": 0.07666166785670459, + "f1_mean": 0.6383589437293336, + "f1_std": 0.08952118234954987 + }, + "ee": { + "accuracy_mean": 0.6341269841269842, + "accuracy_std": 0.06796728274922742, + "precision_mean": 0.583171421347301, + "precision_std": 0.08031795761416705, + "recall_mean": 0.6291817619044511, + "recall_std": 0.10148518503800956, + "f1_mean": 0.5869095341486802, + "f1_std": 0.08729125759225413 + }, + "toxic": { + "accuracy_mean": 0.9445751533791317, + "accuracy_std": 0.020063303116255492, + "precision_mean": 0.7481453634085212, + "precision_std": 0.07540464740954501, + "recall_mean": 0.9548256416904332, + "recall_std": 0.0554955125846467, + "f1_mean": 0.8001151448841631, + "f1_std": 0.039832204457600114 + }, + "biodist": { + "kl_divergence_mean": 0.41026559421902165, + "kl_divergence_std": 0.08303708399525898, + "js_divergence_mean": 0.10230719374124933, + "js_divergence_std": 0.023848212584556397 + } + } +} \ No newline at end of file diff --git a/scripts_run/encode_embeddings/README.md b/scripts_run/encode_embeddings/README.md new file mode 100644 index 0000000..c7d5b74 --- /dev/null +++ b/scripts_run/encode_embeddings/README.md @@ -0,0 +1,43 @@ +# 离线预训练分子 embedding + +三个预训练编码器作为额外的 chemical token 接入模型。embedding 离线生成、 +存为 `.npy`（按 csv 行顺序对齐），训练时通过 `{smiles: vector}` 查表注入。 + +npy 文件与预训练权重不入库，运行下列脚本可复现。 + +## MolFormer (768d) +`ibm/MoLFormer-XL-both-10pct`，HuggingFace 直接加载。 +注意：需 transformers>=4.57（依赖 `transformers.masking_utils`）， +与项目 pin 的 4.45 冲突，建议装到独立目录后用 sys.path 隔离。 +见 `encode_molformer.py`。 + +## GraphMVP (300d) +ICLR'22，5 层 GIN + OGB AtomEncoder/BondEncoder。 +权重：https://github.com/chao1224/GraphMVP → Drive 文件夹 +`GraphMVP_complate_features_for_regression/GraphMVP/pretraining_model.pth` +注意：新版 OGB 的 chirality 类别数为 5（checkpoint 为 4），加载时按形状逐层对齐。 +见 `encode_graphmvp.py`。 + +## GROVER (3200d = atom 1600 + bond 1600) +NeurIPS'20，dualtrans (hidden 800, depth 6)。 +代码：https://github.com/tencent-ailab/grover，权重 grover_base。 +原项目基于 Python 3.6.8 / PyTorch 1.1，在现代环境需两处补丁： + 1. `grover/util/utils.py` 的 `torch.load` 加 `weights_only=False` + 2. `build_model` 前用 checkpoint 自带 args 补齐 current_args 缺失字段， + 并为 dropout 等推理期字段提供默认值 +见 `encode_grover.py`。 + +## 使用 +```bash +python -m lnp_ml.modeling.pretrain main --device cuda \ + --output-dir models/pretrain/ \ + ---emb data/interim/_emb_external.npy \ + ---csv data/external/all_data_LiON.csv + +python -m lnp_ml.modeling.nested_cv_optuna \ + --device cuda --n-trials 15 --epochs-per-trial 30 --n-repeats 3 --seed 42 \ + ---emb data/interim/_emb.npy \ + ---csv data/interim/internal.csv \ + --init-from-pretrain models/pretrain//pretrain_delivery.pt \ + --output-dir models/full/_s42 +``` diff --git a/scripts_run/encode_embeddings/encode_graphmvp.py b/scripts_run/encode_embeddings/encode_graphmvp.py new file mode 100644 index 0000000..477fa5f --- /dev/null +++ b/scripts_run/encode_embeddings/encode_graphmvp.py @@ -0,0 +1,122 @@ +"""GraphMVP 离线 embedding (300d) + +依赖：torch-geometric, ogb +权重：https://github.com/chao1224/GraphMVP → Drive 文件夹内 + GraphMVP_complate_features_for_regression/GraphMVP/pretraining_model.pth + +注意：新版 OGB 的 chirality 类别数为 5，而 checkpoint 为 4， + 加载时按形状逐层对齐（前 4 行用预训练，新增类别保持随机初始化）。 + +用法： + python encode_graphmvp.py --ckpt pretraining_model.pth \ + --csv data/interim/internal.csv --out data/interim/graphmvp_emb.npy +""" +import argparse +import numpy as np +import pandas as pd +import torch +import torch.nn as nn +import torch.nn.functional as F +from ogb.graphproppred.mol_encoder import AtomEncoder, BondEncoder +from ogb.utils.mol import smiles2graph +from torch_geometric.data import Batch, Data +from torch_geometric.nn import MessagePassing, global_mean_pool + + +class GINConv(MessagePassing): + def __init__(self, emb_dim: int) -> None: + super().__init__(aggr="add") + self.mlp = nn.Sequential( + nn.Linear(emb_dim, 2 * emb_dim), nn.BatchNorm1d(2 * emb_dim), + nn.ReLU(), nn.Linear(2 * emb_dim, emb_dim), + ) + self.eps = nn.Parameter(torch.Tensor([0])) + self.bond_encoder = BondEncoder(emb_dim=emb_dim) + + def forward(self, x, edge_index, edge_attr): + e = self.bond_encoder(edge_attr) + return self.mlp((1 + self.eps) * x + self.propagate(edge_index, x=x, edge_attr=e)) + + def message(self, x_j, edge_attr): + return F.relu(x_j + edge_attr) + + def update(self, aggr_out): + return aggr_out + + +class GNNComplete(nn.Module): + def __init__(self, num_layer: int = 5, emb_dim: int = 300, drop_ratio: float = 0.0) -> None: + super().__init__() + self.num_layer, self.drop_ratio = num_layer, drop_ratio + self.atom_encoder = AtomEncoder(emb_dim) + self.gnns = nn.ModuleList([GINConv(emb_dim) for _ in range(num_layer)]) + self.batch_norms = nn.ModuleList([nn.BatchNorm1d(emb_dim) for _ in range(num_layer)]) + + def forward(self, x, edge_index, edge_attr): + h = self.atom_encoder(x) + for layer in range(self.num_layer): + h = self.gnns[layer](h, edge_index, edge_attr) + h = self.batch_norms[layer](h) + h = F.dropout(h if layer == self.num_layer - 1 else F.relu(h), + self.drop_ratio, training=self.training) + return h + + +def load_weights(model: nn.Module, ckpt_path: str) -> None: + sd = torch.load(ckpt_path, map_location="cpu") + msd = model.state_dict() + loaded, skipped, partial = 0, [], [] + for k, v in sd.items(): + if k not in msd: + continue + if msd[k].shape == v.shape: + msd[k] = v + loaded += 1 + elif v.dim() == 2 and msd[k].shape[1] == v.shape[1] and msd[k].shape[0] > v.shape[0]: + msd[k][: v.shape[0]] = v # OGB 类别数变化，前 N 行用预训练 + partial.append(k) + loaded += 1 + else: + skipped.append((k, tuple(v.shape), tuple(msd[k].shape))) + model.load_state_dict(msd) + print(f"已加载 {loaded}/{len(sd)} 个张量 | 部分加载: {partial} | 跳过: {skipped}") + assert not skipped, "有张量形状不兼容" + + +def main(ckpt: str, csv_path: str, out_npy: str, col: str = "smiles", batch: int = 64) -> None: + model = GNNComplete().cuda().eval() + load_weights(model, ckpt) + + smi = pd.read_csv(csv_path, low_memory=False)[col].astype(str).tolist() + uniq = list(dict.fromkeys(smi)) + print(f"{len(smi)} 行, {len(uniq)} 唯一分子") + + lut = {} + with torch.no_grad(): + for i in range(0, len(uniq), batch): + datas = [] + for s in uniq[i:i + batch]: + g = smiles2graph(s) + datas.append(Data( + x=torch.tensor(g["node_feat"], dtype=torch.long), + edge_index=torch.tensor(g["edge_index"], dtype=torch.long), + edge_attr=torch.tensor(g["edge_feat"], dtype=torch.long), + )) + b = Batch.from_data_list(datas).cuda() + hg = global_mean_pool(model(b.x, b.edge_index, b.edge_attr), b.batch).cpu().numpy() + for j, s in enumerate(uniq[i:i + batch]): + lut[s] = hg[j] + + E = np.stack([lut[s] for s in smi]) + np.save(out_npy, E) + print(f"{out_npy} {E.shape}") + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--ckpt", required=True) + p.add_argument("--csv", required=True) + p.add_argument("--out", required=True) + p.add_argument("--col", default="smiles") + a = p.parse_args() + main(a.ckpt, a.csv, a.out, a.col) diff --git a/scripts_run/encode_embeddings/encode_grover.py b/scripts_run/encode_embeddings/encode_grover.py new file mode 100644 index 0000000..a00037d --- /dev/null +++ b/scripts_run/encode_embeddings/encode_grover.py @@ -0,0 +1,57 @@ +"""GROVER fingerprint 离线编码 (3200d = atom 1600 + bond 1600) + +代码：https://github.com/tencent-ailab/grover ，权重 grover_base +原项目基于 Python 3.6.8 / PyTorch 1.1，在现代环境需对 grover 源码打两处补丁： + 1. grover/util/utils.py 的 torch.load 加 weights_only=False + 2. build_model 前用 checkpoint 自带 args 补齐 current_args 缺失字段， + 并为 dropout / features_only / ffn_num_layers 等推理期字段提供默认值 + +三步流程： + # 1) 生成唯一 SMILES 输入 + python encode_grover.py prepare --csv data/interim/internal.csv --out /tmp/grover_in.csv + # 2) 跑 GROVER fingerprint（在 grover 仓库目录下） + python main.py fingerprint --data_path /tmp/grover_in.csv \ + --checkpoint_path grover_base.pt --fingerprint_source both --output /tmp/grover_fp.npz + # 3) 按 csv 行顺序对齐存 npy + python encode_grover.py align --fp /tmp/grover_fp.npz --input /tmp/grover_in.csv \ + --target data/interim/internal.csv --out data/interim/grover_emb.npy +""" +import argparse +import numpy as np +import pandas as pd + + +def prepare(csv_path: str, out_path: str, col: str = "smiles") -> None: + smi = pd.read_csv(csv_path, low_memory=False)[col].astype(str).tolist() + uniq = list(dict.fromkeys(smi)) + pd.DataFrame({"smiles": uniq, "label": [0] * len(uniq)}).to_csv(out_path, index=False) + print(f"{len(smi)} 行 -> {len(uniq)} 唯一分子 -> {out_path}") + + +def align(fp_npz: str, input_csv: str, target_csv: str, out_npy: str, col: str = "smiles") -> None: + fps = np.load(fp_npz)["fps"] + uniq = pd.read_csv(input_csv)["smiles"].astype(str).tolist() + assert len(uniq) == fps.shape[0], f"{len(uniq)} vs {fps.shape[0]}" + lut = {s: fps[i] for i, s in enumerate(uniq)} + rows = pd.read_csv(target_csv, low_memory=False)[col].astype(str).tolist() + E = np.stack([lut[s] for s in rows]) + np.save(out_npy, E) + print(f"{out_npy} {E.shape}") + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + sub = p.add_subparsers(dest="cmd", required=True) + a = sub.add_parser("prepare") + a.add_argument("--csv", required=True) + a.add_argument("--out", required=True) + b = sub.add_parser("align") + b.add_argument("--fp", required=True) + b.add_argument("--input", required=True) + b.add_argument("--target", required=True) + b.add_argument("--out", required=True) + args = p.parse_args() + if args.cmd == "prepare": + prepare(args.csv, args.out) + else: + align(args.fp, args.input, args.target, args.out) diff --git a/scripts_run/encode_embeddings/encode_molformer.py b/scripts_run/encode_embeddings/encode_molformer.py new file mode 100644 index 0000000..793249c --- /dev/null +++ b/scripts_run/encode_embeddings/encode_molformer.py @@ -0,0 +1,51 @@ +"""MolFormer-XL 离线 embedding (768d) + +依赖：transformers>=4.57（需 transformers.masking_utils）。若与项目 pin 的 +4.45 冲突，装到独立目录后用 sys.path 隔离： + pip install --target=/tmp/mf_libs "transformers>=4.57" --no-deps + pip install --target=/tmp/mf_libs tokenizers>=0.22,<=0.23 --no-deps + # 本脚本开头 sys.path.insert(0, '/tmp/mf_libs') + +用法： + python encode_molformer.py --csv data/interim/internal.csv \ + --out data/interim/molformer_emb.npy +""" +import argparse +import numpy as np +import pandas as pd +import torch + +MODEL = "ibm/MoLFormer-XL-both-10pct" + + +def main(csv_path: str, out_npy: str, col: str = "smiles", batch: int = 32) -> None: + from transformers import AutoModel, AutoTokenizer + + smiles = pd.read_csv(csv_path, low_memory=False)[col].astype(str).tolist() + print(f"{len(smiles)} 行, {len(set(smiles))} 唯一分子") + + tok = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=True) + model = AutoModel.from_pretrained( + MODEL, trust_remote_code=True, deterministic_eval=True + ).cuda().eval() + + embs = [] + with torch.no_grad(): + for i in range(0, len(smiles), batch): + b = tok(smiles[i:i + batch], padding=True, truncation=True, + max_length=512, return_tensors="pt") + b = {k: v.cuda() for k, v in b.items()} + embs.append(model(**b).pooler_output.cpu().numpy()) + + E = np.vstack(embs) + np.save(out_npy, E) + print(f"{out_npy} {E.shape}") + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--csv", required=True) + p.add_argument("--out", required=True) + p.add_argument("--col", default="smiles") + a = p.parse_args() + main(a.csv, a.out, a.col)