mirror of
https://github.com/RYDE-WORK/lnp_ml.git
synced 2026-11-17 19:46:07 +08:00
Use fused features for all prediction heads and remove regression bypass
This commit is contained in:
parent
c0d2c008b0
commit
110a524dd2
@ -220,7 +220,7 @@ def train_fold(
|
||||
checkpoint_path = fold_output_dir / "model.pt"
|
||||
torch.save({
|
||||
"model_state_dict": best_state,
|
||||
"config": config,
|
||||
"config": {**config, "head_input": "fused"},
|
||||
"best_val_loss": best_val_loss,
|
||||
"fold_idx": fold_idx,
|
||||
}, checkpoint_path)
|
||||
|
||||
@ -188,13 +188,13 @@ def create_model(
|
||||
moe_top_k: int = 2,
|
||||
moe_expert_hidden_mult: int = 2,
|
||||
moe_jitter_noise: float = 0.0,
|
||||
# LLM(含 reg_bypass / use_rag / soft prompt / QLoRA)
|
||||
# LLM(含 use_rag / soft prompt / QLoRA)
|
||||
llm_kwargs: Optional[Dict] = None,
|
||||
# 检索增强
|
||||
use_retrieval: bool = False,
|
||||
retr_feature_dim: int = 0,
|
||||
) -> Union[LNPModel, LNPModelWithoutMPNN]:
|
||||
"""创建模型。llm_kwargs 含 use_llm / llm_model_path / llm_freeze / llm_use_lora / llm_lora_* / reg_bypass。"""
|
||||
"""创建模型。llm_kwargs 含 use_llm / llm_model_path / llm_freeze / llm_use_lora / llm_lora_*。"""
|
||||
extra_kwargs = dict(
|
||||
set_transformer_block=set_transformer_block,
|
||||
use_moe=use_moe,
|
||||
@ -517,7 +517,6 @@ def main(
|
||||
# Set Transformer
|
||||
set_transformer_block: str = "sab",
|
||||
# 回归旁路
|
||||
reg_bypass: str = "on",
|
||||
# LLM
|
||||
use_llm: bool = False,
|
||||
use_rag: bool = False,
|
||||
@ -560,7 +559,6 @@ def main(
|
||||
raise typer.Exit(1)
|
||||
|
||||
llm_kwargs = dict(
|
||||
reg_bypass=reg_bypass,
|
||||
use_rag=use_rag,
|
||||
rag_top_k=rag_top_k,
|
||||
llm_max_length=llm_max_length,
|
||||
@ -808,7 +806,7 @@ def main(
|
||||
|
||||
torch.save({
|
||||
"model_state_dict": _slim_state,
|
||||
"config": config,
|
||||
"config": {**config, "head_input": "fused"},
|
||||
"best_params": best_params,
|
||||
"epoch_mean": epoch_mean,
|
||||
"use_swa": use_swa,
|
||||
|
||||
@ -101,7 +101,7 @@ class MultiTaskHead(nn.Module):
|
||||
for t in ["size", "delivery", "pdi", "ee", "toxic", "biodist"]
|
||||
})
|
||||
|
||||
def forward(self, x: torch.Tensor, x_numeric: torch.Tensor = None) -> Dict[str, torch.Tensor]:
|
||||
def forward(self, x: torch.Tensor) -> Dict[str, torch.Tensor]:
|
||||
"""
|
||||
Args:
|
||||
x: [B, in_dim] fusion 层输出
|
||||
@ -115,12 +115,11 @@ class MultiTaskHead(nn.Module):
|
||||
- "biodist": [B, 7] probabilities (sum=1)
|
||||
- "toxic": [B, 2] logits
|
||||
"""
|
||||
xn = x if x_numeric is None else x_numeric
|
||||
return {
|
||||
"size": self.size_head(xn),
|
||||
"size": self.size_head(x),
|
||||
"pdi": self.pdi_head(x),
|
||||
"ee": self.ee_head(x),
|
||||
"delivery": self.delivery_head(xn),
|
||||
"delivery": self.delivery_head(x),
|
||||
"biodist": self.biodist_head(x),
|
||||
"toxic": self.toxic_head(x),
|
||||
}
|
||||
|
||||
@ -115,7 +115,6 @@ class LNPModel(nn.Module):
|
||||
moe_expert_hidden_mult: int = 2,
|
||||
moe_jitter_noise: float = 0.0,
|
||||
# ============ 回归旁路开关 ============
|
||||
reg_bypass: str = "on",
|
||||
# ============ LLM 相关 ============
|
||||
use_llm: bool = False,
|
||||
llm_model_path: str = DEFAULT_MOLT5_PATH,
|
||||
@ -232,7 +231,6 @@ class LNPModel(nn.Module):
|
||||
|
||||
# ============ LLM Prompt (可选) ============
|
||||
self.use_llm = use_llm
|
||||
self.reg_bypass = (str(reg_bypass).lower() == "on")
|
||||
if use_llm:
|
||||
self.llm_prompt = LLMPromptEncoder(
|
||||
d_model=d_model,
|
||||
@ -388,12 +386,11 @@ class LNPModel(nn.Module):
|
||||
if task is None:
|
||||
task = "delivery"
|
||||
|
||||
x_reg = getattr(self, "_last_pooled", None) if getattr(self, "reg_bypass", True) else None
|
||||
x_for = {
|
||||
"size": x_reg if x_reg is not None else fused,
|
||||
"size": fused,
|
||||
"pdi": fused,
|
||||
"ee": fused,
|
||||
"delivery": x_reg if x_reg is not None else fused,
|
||||
"delivery": fused,
|
||||
"biodist": fused,
|
||||
"toxic": fused,
|
||||
}
|
||||
@ -447,8 +444,7 @@ class LNPModel(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
"""仅预测 delivery(用于 pretrain)。返回 [B, 1]。"""
|
||||
fused = self.forward_backbone(smiles, tabular)
|
||||
x_reg = getattr(self, "_last_pooled", None) if getattr(self, "reg_bypass", True) else None
|
||||
return self.head.delivery_head(x_reg if x_reg is not None else fused)
|
||||
return self.head.delivery_head(fused)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@ -463,7 +459,7 @@ class LNPModel(nn.Module):
|
||||
delivery [B,1], biodist [B,7], toxic [B,2]
|
||||
"""
|
||||
fused = self.forward_backbone(smiles, tabular)
|
||||
return self.head(fused, getattr(self, "_last_pooled", None) if getattr(self, "reg_bypass", True) else None)
|
||||
return self.head(fused)
|
||||
|
||||
def set_llm_enabled(self, enabled: bool) -> None:
|
||||
"""开关 LLM 旁路,供两阶段推理在粗筛阶段跳过 7B 前向。
|
||||
@ -575,7 +571,6 @@ class LNPModelWithoutMPNN(LNPModel):
|
||||
moe_expert_hidden_mult: int = 2,
|
||||
moe_jitter_noise: float = 0.0,
|
||||
# ============ 回归旁路开关 ============
|
||||
reg_bypass: str = "on",
|
||||
# ============ LLM 相关 ============
|
||||
use_llm: bool = False,
|
||||
llm_model_path: str = DEFAULT_MOLT5_PATH,
|
||||
@ -610,7 +605,6 @@ class LNPModelWithoutMPNN(LNPModel):
|
||||
moleculestm_cache_path=moleculestm_cache_path,
|
||||
mole_cache_path=mole_cache_path,
|
||||
input_dims=dims,
|
||||
reg_bypass=reg_bypass,
|
||||
use_moe=use_moe,
|
||||
moe_n_experts=moe_n_experts,
|
||||
moe_top_k=moe_top_k,
|
||||
|
||||
@ -912,7 +912,7 @@ def _run_single_outer_fold(
|
||||
logger.info(f"保存 checkpoint: {len(_slim_state)} 个张量,其中 LoRA 适配器 {_n_lora} 个")
|
||||
torch.save({
|
||||
"model_state_dict": _slim_state,
|
||||
"config": config,
|
||||
"config": {**config, "head_input": "fused"},
|
||||
"epoch_mean": epoch_mean,
|
||||
"best_params": best_params,
|
||||
}, fold_dir / "model.pt")
|
||||
@ -986,7 +986,6 @@ def main(
|
||||
# Set Transformer
|
||||
set_transformer_block: str = "sab",
|
||||
# 回归旁路(消融开关)
|
||||
reg_bypass: str = "on",
|
||||
# LLM(消融开关)
|
||||
use_llm: bool = False,
|
||||
use_rag: bool = False,
|
||||
@ -1027,7 +1026,6 @@ def main(
|
||||
set_global_seed(seed)
|
||||
|
||||
llm_kwargs = dict(
|
||||
reg_bypass=reg_bypass,
|
||||
use_rag=use_rag,
|
||||
rag_top_k=rag_top_k,
|
||||
llm_max_length=llm_max_length,
|
||||
|
||||
@ -37,10 +37,20 @@ def load_model(
|
||||
加载训练好的模型。
|
||||
|
||||
根据 checkpoint 的 config 决定模型类型与全部结构开关
|
||||
(MoE / LLM / RAG / soft-prompt / reg_bypass)。
|
||||
(MoE / LLM / RAG / soft-prompt)。
|
||||
"""
|
||||
checkpoint = torch.load(model_path, map_location=device, weights_only=False)
|
||||
config = checkpoint["config"]
|
||||
# 新架构显式标记;旧架构只有明确 off 才能按 fused 路径解释。
|
||||
head_input = config.get("head_input")
|
||||
legacy = config.get("reg_bypass")
|
||||
if head_input not in (None, "fused") or (
|
||||
legacy is not None and str(legacy).strip().lower() != "off"
|
||||
) or (head_input is None and legacy is None):
|
||||
raise ValueError(
|
||||
"Checkpoint is not explicitly fused-only. "
|
||||
"Use the original code for legacy or unmarked checkpoints."
|
||||
)
|
||||
use_mpnn = config.get("use_mpnn", False)
|
||||
|
||||
common_kwargs = dict(
|
||||
@ -51,7 +61,6 @@ def load_model(
|
||||
fusion_strategy=config["fusion_strategy"],
|
||||
head_hidden_dim=config["head_hidden_dim"],
|
||||
dropout=config["dropout"],
|
||||
reg_bypass=config.get("reg_bypass", "on"),
|
||||
use_moe=config.get("use_moe", False),
|
||||
moe_n_experts=config.get("moe_n_experts", 4),
|
||||
moe_top_k=config.get("moe_top_k", 2),
|
||||
|
||||
@ -371,6 +371,7 @@ def main(
|
||||
"backbone_state_dict": model.get_backbone_state_dict(),
|
||||
"delivery_head_state_dict": model.get_delivery_head_state_dict(),
|
||||
"config": {
|
||||
"head_input": "fused",
|
||||
"d_model": d_model,
|
||||
"num_heads": num_heads,
|
||||
"n_attn_layers": n_attn_layers,
|
||||
|
||||
@ -14,7 +14,6 @@ from lnp_ml.modeling.nested_cv_optuna import create_model, _build_rag_pool
|
||||
def build_marlin(bp, use_llm):
|
||||
"""use_llm=True -> MARLIN(s1_both); False -> Backbone(s1_baseline)。"""
|
||||
llm_kwargs = dict(
|
||||
reg_bypass="on",
|
||||
use_rag=use_llm, rag_top_k=4, use_llm=use_llm,
|
||||
llm_model_path="models/qwen2.5-7b-instruct",
|
||||
llm_freeze=False, llm_use_lora=False, llm_use_qlora=use_llm,
|
||||
|
||||
@ -1,21 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
SEED=${SEED:-42}
|
||||
MODE=${MODE:-variant} # variant=两卡各跑一个 reg_bypass 变体;shard=两卡分 fold
|
||||
|
||||
if [ "${MODE}" = "shard" ]; then
|
||||
RB=${REG_BYPASS:-off}
|
||||
GPU=0 FOLDS=0,1,2 SEED=${SEED} REG_BYPASS=${RB} BATCH=8 \
|
||||
GPU=0 FOLDS=0,1,2 SEED=${SEED} BATCH=8 \
|
||||
setsid nohup bash scripts_run/run_final_cv.sh >/dev/null 2>&1 &
|
||||
GPU=1 FOLDS=3,4 SEED=${SEED} REG_BYPASS=${RB} BATCH=4 \
|
||||
GPU=1 FOLDS=3,4 SEED=${SEED} BATCH=4 \
|
||||
setsid nohup bash scripts_run/run_final_cv.sh >/dev/null 2>&1 &
|
||||
else
|
||||
GPU=0 SEED=${SEED} REG_BYPASS=off BATCH=8 MIN_FREE_MB=14000 \
|
||||
setsid nohup bash scripts_run/run_final_cv.sh >/dev/null 2>&1 &
|
||||
GPU=1 SEED=${SEED} REG_BYPASS=on BATCH=4 MIN_FREE_MB=11000 \
|
||||
setsid nohup bash scripts_run/run_final_cv.sh >/dev/null 2>&1 &
|
||||
fi
|
||||
|
||||
sleep 2
|
||||
echo "已拉起 MODE=${MODE} SEED=${SEED} PRETRAIN=${PRETRAIN:-<未设置>}"
|
||||
echo "已拉起 MODE=shard SEED=${SEED} PRETRAIN=${PRETRAIN:-<未设置>}"
|
||||
echo "日志:models/final_cv/*/gpu*.log"
|
||||
@ -99,7 +99,6 @@ run_variant baseline || { log "baseline 失败,不再继续 moe_llm"; exit 1;
|
||||
|
||||
run_variant moe_llm \
|
||||
--use-moe --moe-n-experts 4 --moe-top-k 2 \
|
||||
--reg-bypass off \
|
||||
--use-llm --use-rag --rag-top-k 4 \
|
||||
--use-soft-prompt --llm-use-qlora \
|
||||
--llm-model-path models/qwen2.5-7b-instruct \
|
||||
|
||||
@ -6,7 +6,6 @@ set -uo pipefail
|
||||
GPU=${GPU:?必须指定,例如 GPU=0}
|
||||
SEED=${SEED:-42}
|
||||
FOLDS=${FOLDS:-} # 空=跑全部 5 折;"0,1,2"=只跑这三折
|
||||
REG_BYPASS=${REG_BYPASS:-on} # on=delivery/size 绕开 MoE+LLM;off=接入
|
||||
BATCH=${BATCH:-8} # 显存紧就设 4
|
||||
N_OUTER=${N_OUTER:-5}
|
||||
N_INNER=${N_INNER:-3}
|
||||
@ -19,7 +18,7 @@ RETRY_WAIT=${RETRY_WAIT:-300}
|
||||
MIN_OK_SEC=${MIN_OK_SEC:-180} # 存活不足这么久就挂 → 判为配置/代码错误,立即停止重试
|
||||
PRETRAIN=${PRETRAIN:-} # 非空且文件存在才注入 --init-from-pretrain
|
||||
|
||||
RUN_DIR="models/final_cv/${REG_BYPASS}_seed${SEED}"
|
||||
RUN_DIR="models/final_cv/fused_seed${SEED}"
|
||||
TAG="gpu${GPU}$([ -n "${FOLDS}" ] && echo "_folds$(echo "${FOLDS}" | tr -d ',')")"
|
||||
LOG="${RUN_DIR}/${TAG}.log"
|
||||
|
||||
@ -54,7 +53,7 @@ fi
|
||||
|
||||
for attempt in $(seq 1 "${MAX_RETRY}"); do
|
||||
wait_free
|
||||
echo "[$(date '+%F %T')] ATTEMPT ${attempt}/${MAX_RETRY} gpu=${GPU} seed=${SEED} folds=${FOLDS:-all} reg_bypass=${REG_BYPASS} batch=${BATCH} trials=${N_TRIALS}" >>"${LOG}"
|
||||
echo "[$(date '+%F %T')] ATTEMPT ${attempt}/${MAX_RETRY} gpu=${GPU} seed=${SEED} folds=${FOLDS:-all} heads=fused batch=${BATCH} trials=${N_TRIALS}" >>"${LOG}"
|
||||
t0=${SECONDS}
|
||||
|
||||
python -u -m lnp_ml.modeling.nested_cv_optuna \
|
||||
@ -64,7 +63,6 @@ for attempt in $(seq 1 "${MAX_RETRY}"); do
|
||||
--seed ${SEED} \
|
||||
--use-mpnn \
|
||||
--use-moe \
|
||||
--reg-bypass ${REG_BYPASS} \
|
||||
--use-llm --use-rag --rag-top-k 4 \
|
||||
--use-soft-prompt --llm-use-qlora \
|
||||
--llm-model-path models/qwen2.5-7b-instruct \
|
||||
|
||||
@ -14,7 +14,6 @@ N_FOLDS=${N_FOLDS:-3}
|
||||
FIXED_PARAMS=${FIXED_PARAMS:-} # 非空则跳过 Optuna,直接用这份超参
|
||||
FIXED_EPOCH_MEAN=${FIXED_EPOCH_MEAN:-11}
|
||||
BATCH=${BATCH:-8}
|
||||
REG_BYPASS=${REG_BYPASS:-off}
|
||||
FREEZE=${FREEZE:-3}
|
||||
PRETRAIN=${PRETRAIN:-models/pretrain/mpnn/pretrain_delivery.pt}
|
||||
MIN_FREE_MB=${MIN_FREE_MB:-14000}
|
||||
@ -75,7 +74,7 @@ fi
|
||||
for attempt in $(seq 1 "${MAX_RETRY}"); do
|
||||
wait_free
|
||||
NT=$(remaining_trials)
|
||||
echo "[$(date '+%F %T')] ATTEMPT ${attempt}/${MAX_RETRY} gpu=${GPU} 本次补 ${NT} 个 trial(目标 ${N_TRIALS})reg_bypass=${REG_BYPASS} batch=${BATCH}" >>"${LOG}"
|
||||
echo "[$(date '+%F %T')] ATTEMPT ${attempt}/${MAX_RETRY} gpu=${GPU} 本次补 ${NT} 个 trial(目标 ${N_TRIALS})heads=fused batch=${BATCH}" >>"${LOG}"
|
||||
t0=${SECONDS}
|
||||
|
||||
python -u -m lnp_ml.modeling.final_train_optuna_cv \
|
||||
@ -88,7 +87,6 @@ for attempt in $(seq 1 "${MAX_RETRY}"); do
|
||||
--patience ${PATIENCE} \
|
||||
--batch-size ${BATCH} \
|
||||
--use-mpnn --use-moe \
|
||||
--reg-bypass ${REG_BYPASS} \
|
||||
--use-llm --use-rag --rag-top-k 4 \
|
||||
--use-soft-prompt --llm-use-qlora \
|
||||
--llm-model-path models/qwen2.5-7b-instruct \
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user