{
  "dropout": 0.19323529834582587,
  "lr": 0.000980484886752915,
  "weight_decay": 0.00014367509113739864,
  "backbone_lr_ratio": 0.13093310334961422,
  "moe_n_experts": 2,
  "moe_top_k": 2,
  "moe_expert_hidden_mult": 2,
  "llm_lora_r": 8,
  "d_model": 256,
  "num_heads": 8,
  "n_attn_layers": 4,
  "fusion_strategy": "attention",
  "head_hidden_dim": 128,
  "set_transformer_block": "sab"
}