From 110a524dd26aaa8423d5a47a13f7fffc4b8e5b61 Mon Sep 17 00:00:00 2001 From: DicongLi <2024379585@qq.com> Date: Mon, 2 Nov 2026 22:36:39 +0800 Subject: [PATCH] Use fused features for all prediction heads and remove regression bypass --- lnp_ml/modeling/benchmark.py | 2 +- lnp_ml/modeling/final_train_optuna_cv.py | 8 +++----- lnp_ml/modeling/heads.py | 7 +++---- lnp_ml/modeling/models.py | 14 ++++---------- lnp_ml/modeling/nested_cv_optuna.py | 4 +--- lnp_ml/modeling/predict.py | 13 +++++++++++-- lnp_ml/modeling/pretrain.py | 1 + scripts/measure_cost.py | 1 - scripts_run/launch_final_cv.sh | 15 +++------------ scripts_run/run_cv5_sample.sh | 1 - scripts_run/run_final_cv.sh | 6 ++---- scripts_run/run_final_full.sh | 4 +--- 12 files changed, 30 insertions(+), 46 deletions(-) diff --git a/lnp_ml/modeling/benchmark.py b/lnp_ml/modeling/benchmark.py index af9cba0..bd0becb 100644 --- a/lnp_ml/modeling/benchmark.py +++ b/lnp_ml/modeling/benchmark.py @@ -220,7 +220,7 @@ def train_fold( checkpoint_path = fold_output_dir / "model.pt" torch.save({ "model_state_dict": best_state, - "config": config, + "config": {**config, "head_input": "fused"}, "best_val_loss": best_val_loss, "fold_idx": fold_idx, }, checkpoint_path) diff --git a/lnp_ml/modeling/final_train_optuna_cv.py b/lnp_ml/modeling/final_train_optuna_cv.py index 7a94fba..27ab7d3 100644 --- a/lnp_ml/modeling/final_train_optuna_cv.py +++ b/lnp_ml/modeling/final_train_optuna_cv.py @@ -188,13 +188,13 @@ def create_model( moe_top_k: int = 2, moe_expert_hidden_mult: int = 2, moe_jitter_noise: float = 0.0, - # LLM(含 reg_bypass / use_rag / soft prompt / QLoRA) + # LLM(含 use_rag / soft prompt / QLoRA) llm_kwargs: Optional[Dict] = None, # 检索增强 use_retrieval: bool = False, retr_feature_dim: int = 0, ) -> Union[LNPModel, LNPModelWithoutMPNN]: - """创建模型。llm_kwargs 含 use_llm / llm_model_path / llm_freeze / llm_use_lora / llm_lora_* / reg_bypass。""" + """创建模型。llm_kwargs 含 use_llm / llm_model_path / llm_freeze / llm_use_lora / llm_lora_*。""" extra_kwargs = dict( set_transformer_block=set_transformer_block, use_moe=use_moe, @@ -517,7 +517,6 @@ def main( # Set Transformer set_transformer_block: str = "sab", # 回归旁路 - reg_bypass: str = "on", # LLM use_llm: bool = False, use_rag: bool = False, @@ -560,7 +559,6 @@ def main( raise typer.Exit(1) llm_kwargs = dict( - reg_bypass=reg_bypass, use_rag=use_rag, rag_top_k=rag_top_k, llm_max_length=llm_max_length, @@ -808,7 +806,7 @@ def main( torch.save({ "model_state_dict": _slim_state, - "config": config, + "config": {**config, "head_input": "fused"}, "best_params": best_params, "epoch_mean": epoch_mean, "use_swa": use_swa, diff --git a/lnp_ml/modeling/heads.py b/lnp_ml/modeling/heads.py index 329b0ca..347ae95 100644 --- a/lnp_ml/modeling/heads.py +++ b/lnp_ml/modeling/heads.py @@ -101,7 +101,7 @@ class MultiTaskHead(nn.Module): for t in ["size", "delivery", "pdi", "ee", "toxic", "biodist"] }) - def forward(self, x: torch.Tensor, x_numeric: torch.Tensor = None) -> Dict[str, torch.Tensor]: + def forward(self, x: torch.Tensor) -> Dict[str, torch.Tensor]: """ Args: x: [B, in_dim] fusion 层输出 @@ -115,12 +115,11 @@ class MultiTaskHead(nn.Module): - "biodist": [B, 7] probabilities (sum=1) - "toxic": [B, 2] logits """ - xn = x if x_numeric is None else x_numeric return { - "size": self.size_head(xn), + "size": self.size_head(x), "pdi": self.pdi_head(x), "ee": self.ee_head(x), - "delivery": self.delivery_head(xn), + "delivery": self.delivery_head(x), "biodist": self.biodist_head(x), "toxic": self.toxic_head(x), } diff --git a/lnp_ml/modeling/models.py b/lnp_ml/modeling/models.py index b580b32..8660880 100644 --- a/lnp_ml/modeling/models.py +++ b/lnp_ml/modeling/models.py @@ -115,7 +115,6 @@ class LNPModel(nn.Module): moe_expert_hidden_mult: int = 2, moe_jitter_noise: float = 0.0, # ============ 回归旁路开关 ============ - reg_bypass: str = "on", # ============ LLM 相关 ============ use_llm: bool = False, llm_model_path: str = DEFAULT_MOLT5_PATH, @@ -232,7 +231,6 @@ class LNPModel(nn.Module): # ============ LLM Prompt (可选) ============ self.use_llm = use_llm - self.reg_bypass = (str(reg_bypass).lower() == "on") if use_llm: self.llm_prompt = LLMPromptEncoder( d_model=d_model, @@ -388,12 +386,11 @@ class LNPModel(nn.Module): if task is None: task = "delivery" - x_reg = getattr(self, "_last_pooled", None) if getattr(self, "reg_bypass", True) else None x_for = { - "size": x_reg if x_reg is not None else fused, + "size": fused, "pdi": fused, "ee": fused, - "delivery": x_reg if x_reg is not None else fused, + "delivery": fused, "biodist": fused, "toxic": fused, } @@ -447,8 +444,7 @@ class LNPModel(nn.Module): ) -> torch.Tensor: """仅预测 delivery(用于 pretrain)。返回 [B, 1]。""" fused = self.forward_backbone(smiles, tabular) - x_reg = getattr(self, "_last_pooled", None) if getattr(self, "reg_bypass", True) else None - return self.head.delivery_head(x_reg if x_reg is not None else fused) + return self.head.delivery_head(fused) def forward( self, @@ -463,7 +459,7 @@ class LNPModel(nn.Module): delivery [B,1], biodist [B,7], toxic [B,2] """ fused = self.forward_backbone(smiles, tabular) - return self.head(fused, getattr(self, "_last_pooled", None) if getattr(self, "reg_bypass", True) else None) + return self.head(fused) def set_llm_enabled(self, enabled: bool) -> None: """开关 LLM 旁路,供两阶段推理在粗筛阶段跳过 7B 前向。 @@ -575,7 +571,6 @@ class LNPModelWithoutMPNN(LNPModel): moe_expert_hidden_mult: int = 2, moe_jitter_noise: float = 0.0, # ============ 回归旁路开关 ============ - reg_bypass: str = "on", # ============ LLM 相关 ============ use_llm: bool = False, llm_model_path: str = DEFAULT_MOLT5_PATH, @@ -610,7 +605,6 @@ class LNPModelWithoutMPNN(LNPModel): moleculestm_cache_path=moleculestm_cache_path, mole_cache_path=mole_cache_path, input_dims=dims, - reg_bypass=reg_bypass, use_moe=use_moe, moe_n_experts=moe_n_experts, moe_top_k=moe_top_k, diff --git a/lnp_ml/modeling/nested_cv_optuna.py b/lnp_ml/modeling/nested_cv_optuna.py index 64d26ec..2a49cdf 100644 --- a/lnp_ml/modeling/nested_cv_optuna.py +++ b/lnp_ml/modeling/nested_cv_optuna.py @@ -912,7 +912,7 @@ def _run_single_outer_fold( logger.info(f"保存 checkpoint: {len(_slim_state)} 个张量,其中 LoRA 适配器 {_n_lora} 个") torch.save({ "model_state_dict": _slim_state, - "config": config, + "config": {**config, "head_input": "fused"}, "epoch_mean": epoch_mean, "best_params": best_params, }, fold_dir / "model.pt") @@ -986,7 +986,6 @@ def main( # Set Transformer set_transformer_block: str = "sab", # 回归旁路(消融开关) - reg_bypass: str = "on", # LLM(消融开关) use_llm: bool = False, use_rag: bool = False, @@ -1027,7 +1026,6 @@ def main( set_global_seed(seed) llm_kwargs = dict( - reg_bypass=reg_bypass, use_rag=use_rag, rag_top_k=rag_top_k, llm_max_length=llm_max_length, diff --git a/lnp_ml/modeling/predict.py b/lnp_ml/modeling/predict.py index 448f552..22820d3 100644 --- a/lnp_ml/modeling/predict.py +++ b/lnp_ml/modeling/predict.py @@ -37,10 +37,20 @@ def load_model( 加载训练好的模型。 根据 checkpoint 的 config 决定模型类型与全部结构开关 - (MoE / LLM / RAG / soft-prompt / reg_bypass)。 + (MoE / LLM / RAG / soft-prompt)。 """ checkpoint = torch.load(model_path, map_location=device, weights_only=False) config = checkpoint["config"] + # 新架构显式标记;旧架构只有明确 off 才能按 fused 路径解释。 + head_input = config.get("head_input") + legacy = config.get("reg_bypass") + if head_input not in (None, "fused") or ( + legacy is not None and str(legacy).strip().lower() != "off" + ) or (head_input is None and legacy is None): + raise ValueError( + "Checkpoint is not explicitly fused-only. " + "Use the original code for legacy or unmarked checkpoints." + ) use_mpnn = config.get("use_mpnn", False) common_kwargs = dict( @@ -51,7 +61,6 @@ def load_model( fusion_strategy=config["fusion_strategy"], head_hidden_dim=config["head_hidden_dim"], dropout=config["dropout"], - reg_bypass=config.get("reg_bypass", "on"), use_moe=config.get("use_moe", False), moe_n_experts=config.get("moe_n_experts", 4), moe_top_k=config.get("moe_top_k", 2), diff --git a/lnp_ml/modeling/pretrain.py b/lnp_ml/modeling/pretrain.py index 48154ad..4209d38 100644 --- a/lnp_ml/modeling/pretrain.py +++ b/lnp_ml/modeling/pretrain.py @@ -371,6 +371,7 @@ def main( "backbone_state_dict": model.get_backbone_state_dict(), "delivery_head_state_dict": model.get_delivery_head_state_dict(), "config": { + "head_input": "fused", "d_model": d_model, "num_heads": num_heads, "n_attn_layers": n_attn_layers, diff --git a/scripts/measure_cost.py b/scripts/measure_cost.py index 4e112b3..35bc269 100644 --- a/scripts/measure_cost.py +++ b/scripts/measure_cost.py @@ -14,7 +14,6 @@ from lnp_ml.modeling.nested_cv_optuna import create_model, _build_rag_pool def build_marlin(bp, use_llm): """use_llm=True -> MARLIN(s1_both); False -> Backbone(s1_baseline)。""" llm_kwargs = dict( - reg_bypass="on", use_rag=use_llm, rag_top_k=4, use_llm=use_llm, llm_model_path="models/qwen2.5-7b-instruct", llm_freeze=False, llm_use_lora=False, llm_use_qlora=use_llm, diff --git a/scripts_run/launch_final_cv.sh b/scripts_run/launch_final_cv.sh index 7936bad..fe39158 100644 --- a/scripts_run/launch_final_cv.sh +++ b/scripts_run/launch_final_cv.sh @@ -1,21 +1,12 @@ #!/usr/bin/env bash set -uo pipefail SEED=${SEED:-42} -MODE=${MODE:-variant} # variant=两卡各跑一个 reg_bypass 变体;shard=两卡分 fold -if [ "${MODE}" = "shard" ]; then - RB=${REG_BYPASS:-off} - GPU=0 FOLDS=0,1,2 SEED=${SEED} REG_BYPASS=${RB} BATCH=8 \ + GPU=0 FOLDS=0,1,2 SEED=${SEED} BATCH=8 \ setsid nohup bash scripts_run/run_final_cv.sh >/dev/null 2>&1 & - GPU=1 FOLDS=3,4 SEED=${SEED} REG_BYPASS=${RB} BATCH=4 \ + GPU=1 FOLDS=3,4 SEED=${SEED} BATCH=4 \ setsid nohup bash scripts_run/run_final_cv.sh >/dev/null 2>&1 & -else - GPU=0 SEED=${SEED} REG_BYPASS=off BATCH=8 MIN_FREE_MB=14000 \ - setsid nohup bash scripts_run/run_final_cv.sh >/dev/null 2>&1 & - GPU=1 SEED=${SEED} REG_BYPASS=on BATCH=4 MIN_FREE_MB=11000 \ - setsid nohup bash scripts_run/run_final_cv.sh >/dev/null 2>&1 & -fi sleep 2 -echo "已拉起 MODE=${MODE} SEED=${SEED} PRETRAIN=${PRETRAIN:-<未设置>}" +echo "已拉起 MODE=shard SEED=${SEED} PRETRAIN=${PRETRAIN:-<未设置>}" echo "日志:models/final_cv/*/gpu*.log" \ No newline at end of file diff --git a/scripts_run/run_cv5_sample.sh b/scripts_run/run_cv5_sample.sh index 83c6d9b..d7c831f 100644 --- a/scripts_run/run_cv5_sample.sh +++ b/scripts_run/run_cv5_sample.sh @@ -99,7 +99,6 @@ run_variant baseline || { log "baseline 失败,不再继续 moe_llm"; exit 1; run_variant moe_llm \ --use-moe --moe-n-experts 4 --moe-top-k 2 \ - --reg-bypass off \ --use-llm --use-rag --rag-top-k 4 \ --use-soft-prompt --llm-use-qlora \ --llm-model-path models/qwen2.5-7b-instruct \ diff --git a/scripts_run/run_final_cv.sh b/scripts_run/run_final_cv.sh index f370e57..56784ae 100644 --- a/scripts_run/run_final_cv.sh +++ b/scripts_run/run_final_cv.sh @@ -6,7 +6,6 @@ set -uo pipefail GPU=${GPU:?必须指定,例如 GPU=0} SEED=${SEED:-42} FOLDS=${FOLDS:-} # 空=跑全部 5 折;"0,1,2"=只跑这三折 -REG_BYPASS=${REG_BYPASS:-on} # on=delivery/size 绕开 MoE+LLM;off=接入 BATCH=${BATCH:-8} # 显存紧就设 4 N_OUTER=${N_OUTER:-5} N_INNER=${N_INNER:-3} @@ -19,7 +18,7 @@ RETRY_WAIT=${RETRY_WAIT:-300} MIN_OK_SEC=${MIN_OK_SEC:-180} # 存活不足这么久就挂 → 判为配置/代码错误,立即停止重试 PRETRAIN=${PRETRAIN:-} # 非空且文件存在才注入 --init-from-pretrain -RUN_DIR="models/final_cv/${REG_BYPASS}_seed${SEED}" +RUN_DIR="models/final_cv/fused_seed${SEED}" TAG="gpu${GPU}$([ -n "${FOLDS}" ] && echo "_folds$(echo "${FOLDS}" | tr -d ',')")" LOG="${RUN_DIR}/${TAG}.log" @@ -54,7 +53,7 @@ fi for attempt in $(seq 1 "${MAX_RETRY}"); do wait_free - echo "[$(date '+%F %T')] ATTEMPT ${attempt}/${MAX_RETRY} gpu=${GPU} seed=${SEED} folds=${FOLDS:-all} reg_bypass=${REG_BYPASS} batch=${BATCH} trials=${N_TRIALS}" >>"${LOG}" + echo "[$(date '+%F %T')] ATTEMPT ${attempt}/${MAX_RETRY} gpu=${GPU} seed=${SEED} folds=${FOLDS:-all} heads=fused batch=${BATCH} trials=${N_TRIALS}" >>"${LOG}" t0=${SECONDS} python -u -m lnp_ml.modeling.nested_cv_optuna \ @@ -64,7 +63,6 @@ for attempt in $(seq 1 "${MAX_RETRY}"); do --seed ${SEED} \ --use-mpnn \ --use-moe \ - --reg-bypass ${REG_BYPASS} \ --use-llm --use-rag --rag-top-k 4 \ --use-soft-prompt --llm-use-qlora \ --llm-model-path models/qwen2.5-7b-instruct \ diff --git a/scripts_run/run_final_full.sh b/scripts_run/run_final_full.sh index 6376256..b10a49d 100644 --- a/scripts_run/run_final_full.sh +++ b/scripts_run/run_final_full.sh @@ -14,7 +14,6 @@ N_FOLDS=${N_FOLDS:-3} FIXED_PARAMS=${FIXED_PARAMS:-} # 非空则跳过 Optuna,直接用这份超参 FIXED_EPOCH_MEAN=${FIXED_EPOCH_MEAN:-11} BATCH=${BATCH:-8} -REG_BYPASS=${REG_BYPASS:-off} FREEZE=${FREEZE:-3} PRETRAIN=${PRETRAIN:-models/pretrain/mpnn/pretrain_delivery.pt} MIN_FREE_MB=${MIN_FREE_MB:-14000} @@ -75,7 +74,7 @@ fi for attempt in $(seq 1 "${MAX_RETRY}"); do wait_free NT=$(remaining_trials) - echo "[$(date '+%F %T')] ATTEMPT ${attempt}/${MAX_RETRY} gpu=${GPU} 本次补 ${NT} 个 trial(目标 ${N_TRIALS})reg_bypass=${REG_BYPASS} batch=${BATCH}" >>"${LOG}" + echo "[$(date '+%F %T')] ATTEMPT ${attempt}/${MAX_RETRY} gpu=${GPU} 本次补 ${NT} 个 trial(目标 ${N_TRIALS})heads=fused batch=${BATCH}" >>"${LOG}" t0=${SECONDS} python -u -m lnp_ml.modeling.final_train_optuna_cv \ @@ -88,7 +87,6 @@ for attempt in $(seq 1 "${MAX_RETRY}"); do --patience ${PATIENCE} \ --batch-size ${BATCH} \ --use-mpnn --use-moe \ - --reg-bypass ${REG_BYPASS} \ --use-llm --use-rag --rag-top-k 4 \ --use-soft-prompt --llm-use-qlora \ --llm-model-path models/qwen2.5-7b-instruct \