mirror of
https://github.com/RYDE-WORK/lnp_ml.git
synced 2026-09-18 19:13:21 +08:00
16 lines
394 B
JSON
16 lines
394 B
JSON
{
|
|
"dropout": 0.19323529834582587,
|
|
"lr": 0.000980484886752915,
|
|
"weight_decay": 0.00014367509113739864,
|
|
"backbone_lr_ratio": 0.13093310334961422,
|
|
"moe_n_experts": 2,
|
|
"moe_top_k": 2,
|
|
"moe_expert_hidden_mult": 2,
|
|
"llm_lora_r": 8,
|
|
"d_model": 256,
|
|
"num_heads": 8,
|
|
"n_attn_layers": 4,
|
|
"fusion_strategy": "attention",
|
|
"head_hidden_dim": 128,
|
|
"set_transformer_block": "sab"
|
|
} |