Use fused features for all prediction heads and remove regression bypass

This commit is contained in:
DicongLi 2026-11-02 22:36:39 +08:00
parent c0d2c008b0
commit 110a524dd2
12 changed files with 30 additions and 46 deletions

View File

@ -220,7 +220,7 @@ def train_fold(
checkpoint_path = fold_output_dir / "model.pt"
torch.save({
"model_state_dict": best_state,
"config": config,
"config": {**config, "head_input": "fused"},
"best_val_loss": best_val_loss,
"fold_idx": fold_idx,
}, checkpoint_path)

View File

@ -188,13 +188,13 @@ def create_model(
moe_top_k: int = 2,
moe_expert_hidden_mult: int = 2,
moe_jitter_noise: float = 0.0,
# LLM(含 reg_bypass / use_rag / soft prompt / QLoRA)
# LLM(含 use_rag / soft prompt / QLoRA)
llm_kwargs: Optional[Dict] = None,
# 检索增强
use_retrieval: bool = False,
retr_feature_dim: int = 0,
) -> Union[LNPModel, LNPModelWithoutMPNN]:
"""创建模型。llm_kwargs 含 use_llm / llm_model_path / llm_freeze / llm_use_lora / llm_lora_* / reg_bypass。"""
"""创建模型。llm_kwargs 含 use_llm / llm_model_path / llm_freeze / llm_use_lora / llm_lora_*。"""
extra_kwargs = dict(
set_transformer_block=set_transformer_block,
use_moe=use_moe,
@ -517,7 +517,6 @@ def main(
# Set Transformer
set_transformer_block: str = "sab",
# 回归旁路
reg_bypass: str = "on",
# LLM
use_llm: bool = False,
use_rag: bool = False,
@ -560,7 +559,6 @@ def main(
raise typer.Exit(1)
llm_kwargs = dict(
reg_bypass=reg_bypass,
use_rag=use_rag,
rag_top_k=rag_top_k,
llm_max_length=llm_max_length,
@ -808,7 +806,7 @@ def main(
torch.save({
"model_state_dict": _slim_state,
"config": config,
"config": {**config, "head_input": "fused"},
"best_params": best_params,
"epoch_mean": epoch_mean,
"use_swa": use_swa,

View File

@ -101,7 +101,7 @@ class MultiTaskHead(nn.Module):
for t in ["size", "delivery", "pdi", "ee", "toxic", "biodist"]
})
def forward(self, x: torch.Tensor, x_numeric: torch.Tensor = None) -> Dict[str, torch.Tensor]:
def forward(self, x: torch.Tensor) -> Dict[str, torch.Tensor]:
"""
Args:
x: [B, in_dim] fusion 层输出
@ -115,12 +115,11 @@ class MultiTaskHead(nn.Module):
- "biodist": [B, 7] probabilities (sum=1)
- "toxic": [B, 2] logits
"""
xn = x if x_numeric is None else x_numeric
return {
"size": self.size_head(xn),
"size": self.size_head(x),
"pdi": self.pdi_head(x),
"ee": self.ee_head(x),
"delivery": self.delivery_head(xn),
"delivery": self.delivery_head(x),
"biodist": self.biodist_head(x),
"toxic": self.toxic_head(x),
}

View File

@ -115,7 +115,6 @@ class LNPModel(nn.Module):
moe_expert_hidden_mult: int = 2,
moe_jitter_noise: float = 0.0,
# ============ 回归旁路开关 ============
reg_bypass: str = "on",
# ============ LLM 相关 ============
use_llm: bool = False,
llm_model_path: str = DEFAULT_MOLT5_PATH,
@ -232,7 +231,6 @@ class LNPModel(nn.Module):
# ============ LLM Prompt (可选) ============
self.use_llm = use_llm
self.reg_bypass = (str(reg_bypass).lower() == "on")
if use_llm:
self.llm_prompt = LLMPromptEncoder(
d_model=d_model,
@ -388,12 +386,11 @@ class LNPModel(nn.Module):
if task is None:
task = "delivery"
x_reg = getattr(self, "_last_pooled", None) if getattr(self, "reg_bypass", True) else None
x_for = {
"size": x_reg if x_reg is not None else fused,
"size": fused,
"pdi": fused,
"ee": fused,
"delivery": x_reg if x_reg is not None else fused,
"delivery": fused,
"biodist": fused,
"toxic": fused,
}
@ -447,8 +444,7 @@ class LNPModel(nn.Module):
) -> torch.Tensor:
"""仅预测 delivery(用于 pretrain)。返回 [B, 1]。"""
fused = self.forward_backbone(smiles, tabular)
x_reg = getattr(self, "_last_pooled", None) if getattr(self, "reg_bypass", True) else None
return self.head.delivery_head(x_reg if x_reg is not None else fused)
return self.head.delivery_head(fused)
def forward(
self,
@ -463,7 +459,7 @@ class LNPModel(nn.Module):
delivery [B,1], biodist [B,7], toxic [B,2]
"""
fused = self.forward_backbone(smiles, tabular)
return self.head(fused, getattr(self, "_last_pooled", None) if getattr(self, "reg_bypass", True) else None)
return self.head(fused)
def set_llm_enabled(self, enabled: bool) -> None:
"""开关 LLM 旁路,供两阶段推理在粗筛阶段跳过 7B 前向。
@ -575,7 +571,6 @@ class LNPModelWithoutMPNN(LNPModel):
moe_expert_hidden_mult: int = 2,
moe_jitter_noise: float = 0.0,
# ============ 回归旁路开关 ============
reg_bypass: str = "on",
# ============ LLM 相关 ============
use_llm: bool = False,
llm_model_path: str = DEFAULT_MOLT5_PATH,
@ -610,7 +605,6 @@ class LNPModelWithoutMPNN(LNPModel):
moleculestm_cache_path=moleculestm_cache_path,
mole_cache_path=mole_cache_path,
input_dims=dims,
reg_bypass=reg_bypass,
use_moe=use_moe,
moe_n_experts=moe_n_experts,
moe_top_k=moe_top_k,

View File

@ -912,7 +912,7 @@ def _run_single_outer_fold(
logger.info(f"保存 checkpoint: {len(_slim_state)} 个张量,其中 LoRA 适配器 {_n_lora} 个")
torch.save({
"model_state_dict": _slim_state,
"config": config,
"config": {**config, "head_input": "fused"},
"epoch_mean": epoch_mean,
"best_params": best_params,
}, fold_dir / "model.pt")
@ -986,7 +986,6 @@ def main(
# Set Transformer
set_transformer_block: str = "sab",
# 回归旁路(消融开关)
reg_bypass: str = "on",
# LLM(消融开关)
use_llm: bool = False,
use_rag: bool = False,
@ -1027,7 +1026,6 @@ def main(
set_global_seed(seed)
llm_kwargs = dict(
reg_bypass=reg_bypass,
use_rag=use_rag,
rag_top_k=rag_top_k,
llm_max_length=llm_max_length,

View File

@ -37,10 +37,20 @@ def load_model(
加载训练好的模型。
根据 checkpoint 的 config 决定模型类型与全部结构开关
(MoE / LLM / RAG / soft-prompt / reg_bypass)。
(MoE / LLM / RAG / soft-prompt)。
"""
checkpoint = torch.load(model_path, map_location=device, weights_only=False)
config = checkpoint["config"]
# 新架构显式标记;旧架构只有明确 off 才能按 fused 路径解释。
head_input = config.get("head_input")
legacy = config.get("reg_bypass")
if head_input not in (None, "fused") or (
legacy is not None and str(legacy).strip().lower() != "off"
) or (head_input is None and legacy is None):
raise ValueError(
"Checkpoint is not explicitly fused-only. "
"Use the original code for legacy or unmarked checkpoints."
)
use_mpnn = config.get("use_mpnn", False)
common_kwargs = dict(
@ -51,7 +61,6 @@ def load_model(
fusion_strategy=config["fusion_strategy"],
head_hidden_dim=config["head_hidden_dim"],
dropout=config["dropout"],
reg_bypass=config.get("reg_bypass", "on"),
use_moe=config.get("use_moe", False),
moe_n_experts=config.get("moe_n_experts", 4),
moe_top_k=config.get("moe_top_k", 2),

View File

@ -371,6 +371,7 @@ def main(
"backbone_state_dict": model.get_backbone_state_dict(),
"delivery_head_state_dict": model.get_delivery_head_state_dict(),
"config": {
"head_input": "fused",
"d_model": d_model,
"num_heads": num_heads,
"n_attn_layers": n_attn_layers,

View File

@ -14,7 +14,6 @@ from lnp_ml.modeling.nested_cv_optuna import create_model, _build_rag_pool
def build_marlin(bp, use_llm):
"""use_llm=True -> MARLIN(s1_both); False -> Backbone(s1_baseline)。"""
llm_kwargs = dict(
reg_bypass="on",
use_rag=use_llm, rag_top_k=4, use_llm=use_llm,
llm_model_path="models/qwen2.5-7b-instruct",
llm_freeze=False, llm_use_lora=False, llm_use_qlora=use_llm,

View File

@ -1,21 +1,12 @@
#!/usr/bin/env bash
set -uo pipefail
SEED=${SEED:-42}
MODE=${MODE:-variant} # variant=两卡各跑一个 reg_bypass 变体;shard=两卡分 fold
if [ "${MODE}" = "shard" ]; then
RB=${REG_BYPASS:-off}
GPU=0 FOLDS=0,1,2 SEED=${SEED} REG_BYPASS=${RB} BATCH=8 \
GPU=0 FOLDS=0,1,2 SEED=${SEED} BATCH=8 \
setsid nohup bash scripts_run/run_final_cv.sh >/dev/null 2>&1 &
GPU=1 FOLDS=3,4 SEED=${SEED} REG_BYPASS=${RB} BATCH=4 \
GPU=1 FOLDS=3,4 SEED=${SEED} BATCH=4 \
setsid nohup bash scripts_run/run_final_cv.sh >/dev/null 2>&1 &
else
GPU=0 SEED=${SEED} REG_BYPASS=off BATCH=8 MIN_FREE_MB=14000 \
setsid nohup bash scripts_run/run_final_cv.sh >/dev/null 2>&1 &
GPU=1 SEED=${SEED} REG_BYPASS=on BATCH=4 MIN_FREE_MB=11000 \
setsid nohup bash scripts_run/run_final_cv.sh >/dev/null 2>&1 &
fi
sleep 2
echo "已拉起 MODE=${MODE} SEED=${SEED} PRETRAIN=${PRETRAIN:-<未设置>}"
echo "已拉起 MODE=shard SEED=${SEED} PRETRAIN=${PRETRAIN:-<未设置>}"
echo "日志:models/final_cv/*/gpu*.log"

View File

@ -99,7 +99,6 @@ run_variant baseline || { log "baseline 失败,不再继续 moe_llm"; exit 1;
run_variant moe_llm \
--use-moe --moe-n-experts 4 --moe-top-k 2 \
--reg-bypass off \
--use-llm --use-rag --rag-top-k 4 \
--use-soft-prompt --llm-use-qlora \
--llm-model-path models/qwen2.5-7b-instruct \

View File

@ -6,7 +6,6 @@ set -uo pipefail
GPU=${GPU:?必须指定,例如 GPU=0}
SEED=${SEED:-42}
FOLDS=${FOLDS:-} # 空=跑全部 5 折;"0,1,2"=只跑这三折
REG_BYPASS=${REG_BYPASS:-on} # on=delivery/size 绕开 MoE+LLM;off=接入
BATCH=${BATCH:-8} # 显存紧就设 4
N_OUTER=${N_OUTER:-5}
N_INNER=${N_INNER:-3}
@ -19,7 +18,7 @@ RETRY_WAIT=${RETRY_WAIT:-300}
MIN_OK_SEC=${MIN_OK_SEC:-180} # 存活不足这么久就挂 → 判为配置/代码错误,立即停止重试
PRETRAIN=${PRETRAIN:-} # 非空且文件存在才注入 --init-from-pretrain
RUN_DIR="models/final_cv/${REG_BYPASS}_seed${SEED}"
RUN_DIR="models/final_cv/fused_seed${SEED}"
TAG="gpu${GPU}$([ -n "${FOLDS}" ] && echo "_folds$(echo "${FOLDS}" | tr -d ',')")"
LOG="${RUN_DIR}/${TAG}.log"
@ -54,7 +53,7 @@ fi
for attempt in $(seq 1 "${MAX_RETRY}"); do
wait_free
echo "[$(date '+%F %T')] ATTEMPT ${attempt}/${MAX_RETRY} gpu=${GPU} seed=${SEED} folds=${FOLDS:-all} reg_bypass=${REG_BYPASS} batch=${BATCH} trials=${N_TRIALS}" >>"${LOG}"
echo "[$(date '+%F %T')] ATTEMPT ${attempt}/${MAX_RETRY} gpu=${GPU} seed=${SEED} folds=${FOLDS:-all} heads=fused batch=${BATCH} trials=${N_TRIALS}" >>"${LOG}"
t0=${SECONDS}
python -u -m lnp_ml.modeling.nested_cv_optuna \
@ -64,7 +63,6 @@ for attempt in $(seq 1 "${MAX_RETRY}"); do
--seed ${SEED} \
--use-mpnn \
--use-moe \
--reg-bypass ${REG_BYPASS} \
--use-llm --use-rag --rag-top-k 4 \
--use-soft-prompt --llm-use-qlora \
--llm-model-path models/qwen2.5-7b-instruct \

View File

@ -14,7 +14,6 @@ N_FOLDS=${N_FOLDS:-3}
FIXED_PARAMS=${FIXED_PARAMS:-} # 非空则跳过 Optuna,直接用这份超参
FIXED_EPOCH_MEAN=${FIXED_EPOCH_MEAN:-11}
BATCH=${BATCH:-8}
REG_BYPASS=${REG_BYPASS:-off}
FREEZE=${FREEZE:-3}
PRETRAIN=${PRETRAIN:-models/pretrain/mpnn/pretrain_delivery.pt}
MIN_FREE_MB=${MIN_FREE_MB:-14000}
@ -75,7 +74,7 @@ fi
for attempt in $(seq 1 "${MAX_RETRY}"); do
wait_free
NT=$(remaining_trials)
echo "[$(date '+%F %T')] ATTEMPT ${attempt}/${MAX_RETRY} gpu=${GPU} 本次补 ${NT} 个 trial(目标 ${N_TRIALS})reg_bypass=${REG_BYPASS} batch=${BATCH}" >>"${LOG}"
echo "[$(date '+%F %T')] ATTEMPT ${attempt}/${MAX_RETRY} gpu=${GPU} 本次补 ${NT} 个 trial(目标 ${N_TRIALS})heads=fused batch=${BATCH}" >>"${LOG}"
t0=${SECONDS}
python -u -m lnp_ml.modeling.final_train_optuna_cv \
@ -88,7 +87,6 @@ for attempt in $(seq 1 "${MAX_RETRY}"); do
--patience ${PATIENCE} \
--batch-size ${BATCH} \
--use-mpnn --use-moe \
--reg-bypass ${REG_BYPASS} \
--use-llm --use-rag --rag-top-k 4 \
--use-soft-prompt --llm-use-qlora \
--llm-model-path models/qwen2.5-7b-instruct \