diff --git a/models/cv5_sample/baseline/seed42/outer_fold_0/best_params.json b/models/cv5_sample/baseline/seed42/outer_fold_0/best_params.json new file mode 100644 index 0000000..df17e17 --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_0/best_params.json @@ -0,0 +1,16 @@ +{ + "dropout": 0.19323529834582587, + "lr": 0.000980484886752915, + "weight_decay": 0.00014367509113739864, + "backbone_lr_ratio": 0.13093310334961422, + "moe_n_experts": 2, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_0/epoch_mean.json b/models/cv5_sample/baseline/seed42/outer_fold_0/epoch_mean.json new file mode 100644 index 0000000..735a6b9 --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_0/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 11} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_0/history.json b/models/cv5_sample/baseline/seed42/outer_fold_0/history.json new file mode 100644 index 0000000..e326b16 --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_0/history.json @@ -0,0 +1,237 @@ +{ + "train": [ + { + "loss": 5.055936246304898, + "loss_size": 0.8606430002160974, + "loss_pdi": 0.6768325905542116, + "loss_ee": 1.0347085675677739, + "loss_delivery": 1.0286872987006161, + "loss_biodist": 0.9479167284192266, + "loss_toxic": 0.52346063224045 + }, + { + "loss": 3.99745157602671, + "loss_size": 0.7864497557685182, + "loss_pdi": 0.6331288669560406, + "loss_ee": 0.9356951681343285, + "loss_delivery": 0.8529001963702408, + "loss_biodist": 0.4785909547999099, + "loss_toxic": 0.33874212809511134 + }, + { + "loss": 3.6507615302060104, + "loss_size": 0.7465558921968615, + "loss_pdi": 0.5953955263704866, + "loss_ee": 0.8915211453631118, + "loss_delivery": 0.8915189435151784, + "loss_biodist": 0.3364428563697918, + "loss_toxic": 0.23094557698916746 + }, + { + "loss": 3.6875334591478914, + "loss_size": 0.7882098569660574, + "loss_pdi": 0.5826210242670935, + "loss_ee": 0.8787951453312023, + "loss_delivery": 1.0553698157136504, + "loss_biodist": 0.24477992005444862, + "loss_toxic": 0.19957500592981642 + }, + { + "loss": 3.2242523431777954, + "loss_size": 0.7434807216396203, + "loss_pdi": 0.5325888967191851, + "loss_ee": 0.8109645078311095, + "loss_delivery": 0.8456436809454415, + "loss_biodist": 0.2152977317974374, + "loss_toxic": 0.13414243228632855 + }, + { + "loss": 2.72972969428913, + "loss_size": 0.41291164667219726, + "loss_pdi": 0.5447176517666997, + "loss_ee": 0.7833192106839773, + "loss_delivery": 0.7865552172266148, + "loss_biodist": 0.18164325535700127, + "loss_toxic": 0.08577694238251629 + }, + { + "loss": 2.7861866564364046, + "loss_size": 0.5654866753800495, + "loss_pdi": 0.4972686191668382, + "loss_ee": 0.748628426242519, + "loss_delivery": 0.7679579907373802, + "loss_biodist": 0.2135133976871903, + "loss_toxic": 0.05895325155123263 + }, + { + "loss": 2.7905788904911764, + "loss_size": 0.5844308375506788, + "loss_pdi": 0.5185149493249687, + "loss_ee": 0.7592554019915091, + "loss_delivery": 0.7543600313365459, + "loss_biodist": 0.16718868860924566, + "loss_toxic": 0.0781663300664592 + }, + { + "loss": 2.7408436472351485, + "loss_size": 0.4887551304859084, + "loss_pdi": 0.5093961752749778, + "loss_ee": 0.7876343791549271, + "loss_delivery": 0.7577200457453728, + "loss_biodist": 0.18930391161828428, + "loss_toxic": 0.08456783540660164 + }, + { + "loss": 2.4752910588238692, + "loss_size": 0.4774871263149622, + "loss_pdi": 0.46845414630464605, + "loss_ee": 0.7078113322322434, + "loss_delivery": 0.6665648940849949, + "loss_biodist": 0.16393172116698446, + "loss_toxic": 0.05852524563040886 + }, + { + "loss": 2.5551916940792188, + "loss_size": 0.5141892947659299, + "loss_pdi": 0.4798985337083404, + "loss_ee": 0.7492339151936609, + "loss_delivery": 0.6760205814640063, + "loss_biodist": 0.14918651633165977, + "loss_toxic": 0.060266225545264375 + } + ], + "val": [ + { + "loss": 3.9572858810424805, + "loss_size": 0.5028525212779641, + "loss_pdi": 0.7401092886924744, + "loss_ee": 0.9889105439186097, + "loss_delivery": 0.6921922609210014, + "loss_biodist": 0.7097499012947083, + "loss_toxic": 0.35348092019557953, + "acc_pdi": 0.6060606060606061, + "acc_ee": 0.48484848484848486, + "acc_toxic": 0.95 + }, + { + "loss": 3.4830109119415282, + "loss_size": 0.4342193953692913, + "loss_pdi": 0.81399564743042, + "loss_ee": 0.8462728559970856, + "loss_delivery": 0.598291178047657, + "loss_biodist": 0.5577272236347198, + "loss_toxic": 0.2763485822826624, + "acc_pdi": 0.5757575757575758, + "acc_ee": 0.5151515151515151, + "acc_toxic": 0.9 + }, + { + "loss": 3.4455579280853272, + "loss_size": 0.39207540452480316, + "loss_pdi": 0.8946391701698303, + "loss_ee": 0.8172489017248153, + "loss_delivery": 0.6878725722432136, + "loss_biodist": 0.3735358417034149, + "loss_toxic": 0.3407814013771713, + "acc_pdi": 0.6363636363636364, + "acc_ee": 0.48484848484848486, + "acc_toxic": 0.9 + }, + { + "loss": 4.0154601573944095, + "loss_size": 0.4502963304519653, + "loss_pdi": 1.2591865181922912, + "loss_ee": 0.8082401096820832, + "loss_delivery": 0.6105723276734352, + "loss_biodist": 0.29027132838964465, + "loss_toxic": 0.6681207884568721, + "acc_pdi": 0.3939393939393939, + "acc_ee": 0.48484848484848486, + "acc_toxic": 0.85 + }, + { + "loss": 3.352661943435669, + "loss_size": 0.32106279730796816, + "loss_pdi": 1.0119264781475068, + "loss_ee": 0.9331188946962357, + "loss_delivery": 0.5763307940214872, + "loss_biodist": 0.21531529426574708, + "loss_toxic": 0.37886948613449933, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.45454545454545453, + "acc_toxic": 0.9 + }, + { + "loss": 3.444174289703369, + "loss_size": 0.35679062008857726, + "loss_pdi": 1.155838769674301, + "loss_ee": 0.8481447480618953, + "loss_delivery": 0.6408442467451095, + "loss_biodist": 0.17143512666225433, + "loss_toxic": 0.36500417003408075, + "acc_pdi": 0.6363636363636364, + "acc_ee": 0.45454545454545453, + "acc_toxic": 0.9 + }, + { + "loss": 3.0015304565429686, + "loss_size": 0.3286006212234497, + "loss_pdi": 1.0518546760082246, + "loss_ee": 0.8692154731601477, + "loss_delivery": 0.5558455713093281, + "loss_biodist": 0.15108025521039964, + "loss_toxic": 0.15213317363522946, + "acc_pdi": 0.6060606060606061, + "acc_ee": 0.48484848484848486, + "acc_toxic": 1.0 + }, + { + "loss": 3.3050911903381346, + "loss_size": 0.3655768930912018, + "loss_pdi": 1.2608545422554016, + "loss_ee": 0.8624444365501404, + "loss_delivery": 0.5735122933983803, + "loss_biodist": 0.1565265588462353, + "loss_toxic": 0.20591226250398903, + "acc_pdi": 0.5454545454545454, + "acc_ee": 0.45454545454545453, + "acc_toxic": 0.9 + }, + { + "loss": 3.1276957035064696, + "loss_size": 0.30587572753429415, + "loss_pdi": 1.2177553713321685, + "loss_ee": 0.8245017357170582, + "loss_delivery": 0.5470387808978557, + "loss_biodist": 0.13348158523440362, + "loss_toxic": 0.2128354288637638, + "acc_pdi": 0.45454545454545453, + "acc_ee": 0.45454545454545453, + "acc_toxic": 0.9 + }, + { + "loss": 3.1956005096435547, + "loss_size": 0.3409146010875702, + "loss_pdi": 1.250557976961136, + "loss_ee": 0.8290754735469819, + "loss_delivery": 0.5627829760313035, + "loss_biodist": 0.12690359950065613, + "loss_toxic": 0.20395291885361075, + "acc_pdi": 0.5151515151515151, + "acc_ee": 0.45454545454545453, + "acc_toxic": 0.9 + }, + { + "loss": 3.1633092880249025, + "loss_size": 0.35122549831867217, + "loss_pdi": 1.2853083610534668, + "loss_ee": 0.7996505998075009, + "loss_delivery": 0.579404155164957, + "loss_biodist": 0.13538600280880927, + "loss_toxic": 0.13744947910308838, + "acc_pdi": 0.5151515151515151, + "acc_ee": 0.48484848484848486, + "acc_toxic": 1.0 + } + ] +} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_0/splits.json b/models/cv5_sample/baseline/seed42/outer_fold_0/splits.json new file mode 100644 index 0000000..cccb9d8 --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_0/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 52, 53, 54, 55, 56, 57, 58, 61, 62, 63, 64, 65, 66, 67, 68, 71, 72, 73, 75, 76, 77, 80, 81, 85, 86, 88, 89, 90, 92, 94, 95, 96, 98, 99, 100, 101, 103, 104, 105, 106, 107, 108, 109, 111, 113, 114, 115, 116, 117, 118, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 133, 134, 135, 136, 138, 139, 141, 144, 145, 146, 147, 148, 149, 150, 151, 153, 154, 155, 156, 157, 158, 159, 160, 163, 164, 167, 168, 169, 170, 171, 172, 175, 176, 178, 179, 180, 182, 183, 184, 185, 188, 189, 190, 191, 192, 193, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 206, 208, 210, 211, 212, 213, 216, 217, 218, 219, 220, 222, 224, 225, 227, 228, 229, 230, 231, 232, 233, 234, 236, 237, 239, 241, 242, 244, 245, 247, 248, 249, 250, 252, 253, 255, 256, 257, 258, 259, 260, 261, 262, 265, 267, 268, 269, 270, 271, 272, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 285, 286, 287, 288, 291, 292, 294, 296, 297, 298, 299, 300, 301, 303, 304, 306, 307, 308, 309, 310, 311, 312, 313, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 332, 333, 335, 336, 337, 338, 339, 340, 342, 343, 344, 345, 346, 347, 348, 349, 351, 352, 354, 355, 356, 357, 358, 359, 360, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 374, 375, 377, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 393, 395, 396, 397, 398, 399, 400, 401, 403, 404, 405, 406, 407, 408, 409, 411, 412, 413, 414, 415, 417, 418, 419], "outer_test_idx": [7, 16, 20, 29, 51, 59, 60, 69, 70, 74, 78, 79, 82, 83, 84, 87, 91, 93, 97, 102, 110, 112, 119, 120, 121, 132, 137, 140, 142, 143, 152, 161, 162, 165, 166, 173, 174, 177, 181, 186, 187, 194, 205, 207, 209, 214, 215, 221, 223, 226, 235, 238, 240, 243, 246, 251, 254, 263, 264, 266, 273, 284, 289, 290, 293, 295, 302, 305, 314, 315, 331, 334, 341, 350, 353, 361, 373, 376, 378, 392, 394, 402, 410, 416]} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_0/test_metrics.json b/models/cv5_sample/baseline/seed42/outer_fold_0/test_metrics.json new file mode 100644 index 0000000..b7baf3a --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_0/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 83, + "mse": 1.5968842484584136, + "rmse": 1.263678854954222, + "mae": 0.5682635198246284, + "r2": 0.022667532793975265 + }, + "delivery": { + "n_samples": 58, + "mse": 0.5483822645024997, + "rmse": 0.740528368465719, + "mae": 0.5250802371843621, + "r2": 0.3029730502833111 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7142857142857143, + "precision": 0.6737967914438503, + "recall": 0.7084818246614397, + "f1": 0.6785714285714286 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7380952380952381, + "precision": 0.7073879551820729, + "recall": 0.7647619047619049, + "f1": 0.7124896608767578 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9827586206896551, + "precision": 0.9912280701754386, + "recall": 0.75, + "f1": 0.8289085545722714 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.1971223395298808, + "js_divergence": 0.043043902163233595 + } +} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_1/best_params.json b/models/cv5_sample/baseline/seed42/outer_fold_1/best_params.json new file mode 100644 index 0000000..df17e17 --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_1/best_params.json @@ -0,0 +1,16 @@ +{ + "dropout": 0.19323529834582587, + "lr": 0.000980484886752915, + "weight_decay": 0.00014367509113739864, + "backbone_lr_ratio": 0.13093310334961422, + "moe_n_experts": 2, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_1/epoch_mean.json b/models/cv5_sample/baseline/seed42/outer_fold_1/epoch_mean.json new file mode 100644 index 0000000..735a6b9 --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_1/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 11} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_1/history.json b/models/cv5_sample/baseline/seed42/outer_fold_1/history.json new file mode 100644 index 0000000..a6a1d5f --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_1/history.json @@ -0,0 +1,237 @@ +{ + "train": [ + { + "loss": 5.3353071212768555, + "loss_size": 1.2256073724176433, + "loss_pdi": 0.6600620666065732, + "loss_ee": 1.020295173735232, + "loss_delivery": 0.9249667110475334, + "loss_biodist": 0.9719328582286835, + "loss_toxic": 0.5454153837384405 + }, + { + "loss": 4.285548815856108, + "loss_size": 1.111898030179578, + "loss_pdi": 0.6168666175893835, + "loss_ee": 0.9040256867537627, + "loss_delivery": 0.9542275908831004, + "loss_biodist": 0.4325796188534917, + "loss_toxic": 0.29354191521132317 + }, + { + "loss": 3.56095738024325, + "loss_size": 0.7991758820575636, + "loss_pdi": 0.5580040844711097, + "loss_ee": 0.8332083772968601, + "loss_delivery": 0.9466991460806614, + "loss_biodist": 0.27789403236395605, + "loss_toxic": 0.18792380020022392 + }, + { + "loss": 3.5710522806322254, + "loss_size": 1.046761174661082, + "loss_pdi": 0.5474661910856092, + "loss_ee": 0.8386256227622161, + "loss_delivery": 0.758317036201825, + "loss_biodist": 0.23423250090028788, + "loss_toxic": 0.19732182759892297 + }, + { + "loss": 3.344043564152073, + "loss_size": 0.9940445900366113, + "loss_pdi": 0.5329670773164646, + "loss_ee": 0.7994254424765304, + "loss_delivery": 0.7578543800910985, + "loss_biodist": 0.2094646818130403, + "loss_toxic": 0.1107913074335335 + }, + { + "loss": 2.9736292458869316, + "loss_size": 0.9058008608785836, + "loss_pdi": 0.5148797063408671, + "loss_ee": 0.7447971218341106, + "loss_delivery": 0.6041139666670682, + "loss_biodist": 0.19747554249054677, + "loss_toxic": 0.06619761599152274 + }, + { + "loss": 3.091302794379157, + "loss_size": 0.8345884137056969, + "loss_pdi": 0.4844536020143612, + "loss_ee": 0.7188561582887495, + "loss_delivery": 0.6844744932007145, + "loss_biodist": 0.16666732728481293, + "loss_toxic": 0.24889929544784734 + }, + { + "loss": 2.474903399879868, + "loss_size": 0.4893314063146308, + "loss_pdi": 0.4681552906294127, + "loss_ee": 0.699250686007577, + "loss_delivery": 0.6240661612434967, + "loss_biodist": 0.181533552202824, + "loss_toxic": 0.07646923640906508 + }, + { + "loss": 2.7190622445699333, + "loss_size": 0.7405592789118355, + "loss_pdi": 0.4723592585808522, + "loss_ee": 0.6855430079473032, + "loss_delivery": 0.6662846245878452, + "loss_biodist": 0.1865380845359854, + "loss_toxic": 0.03832938376738614 + }, + { + "loss": 2.7083647992159867, + "loss_size": 0.7273675418786101, + "loss_pdi": 0.46286967477282964, + "loss_ee": 0.6985194336723637, + "loss_delivery": 0.634993327781558, + "loss_biodist": 0.16010415433226405, + "loss_toxic": 0.09129857794834755 + }, + { + "loss": 2.5737238510234937, + "loss_size": 0.7433734733510662, + "loss_pdi": 0.46628197262416016, + "loss_ee": 0.668087279474413, + "loss_delivery": 0.5395700915439708, + "loss_biodist": 0.1628361604100949, + "loss_toxic": 0.05862165063720297 + } + ], + "val": [ + { + "loss": 3.031777000427246, + "loss_size": 0.5401369854807854, + "loss_pdi": 0.6682288587093353, + "loss_ee": 0.9218176007270813, + "loss_delivery": 0.2300173431634903, + "loss_biodist": 0.5083122372627258, + "loss_toxic": 0.17078153491020204, + "acc_pdi": 0.6363636363636364, + "acc_ee": 0.696969696969697, + "acc_toxic": 1.0 + }, + { + "loss": 2.502068042755127, + "loss_size": 0.48314091861248015, + "loss_pdi": 0.639093154668808, + "loss_ee": 0.6983676433563233, + "loss_delivery": 0.3520810127258301, + "loss_biodist": 0.2895366489887238, + "loss_toxic": 0.052681690081954, + "acc_pdi": 0.6363636363636364, + "acc_ee": 0.7575757575757576, + "acc_toxic": 1.0 + }, + { + "loss": 2.417718243598938, + "loss_size": 0.44972713887691496, + "loss_pdi": 0.6729098677635192, + "loss_ee": 0.6924970507621765, + "loss_delivery": 0.36018016636371614, + "loss_biodist": 0.26438169181346893, + "loss_toxic": 0.019849006040021776, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.7272727272727273, + "acc_toxic": 1.0 + }, + { + "loss": 2.685157823562622, + "loss_size": 0.4716562867164612, + "loss_pdi": 0.5998583734035492, + "loss_ee": 0.601423978805542, + "loss_delivery": 0.7582527756690979, + "loss_biodist": 0.2994823604822159, + "loss_toxic": 0.007456492912024259, + "acc_pdi": 0.6060606060606061, + "acc_ee": 0.7878787878787878, + "acc_toxic": 1.0 + }, + { + "loss": 2.0070614576339723, + "loss_size": 0.44010728001594546, + "loss_pdi": 0.6203509151935578, + "loss_ee": 0.6338537752628326, + "loss_delivery": 0.1313400834798813, + "loss_biodist": 0.22033900320529937, + "loss_toxic": 0.004034112859517336, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.7878787878787878, + "acc_toxic": 1.0 + }, + { + "loss": 2.1099953651428223, + "loss_size": 0.4530618578195572, + "loss_pdi": 0.6262755393981934, + "loss_ee": 0.597742722928524, + "loss_delivery": 0.30005890130996704, + "loss_biodist": 0.18362512215971946, + "loss_toxic": 0.0021228168159723284, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.7575757575757576, + "acc_toxic": 1.0 + }, + { + "loss": 2.2114261865615843, + "loss_size": 0.5167493343353271, + "loss_pdi": 0.5672660648822785, + "loss_ee": 0.6098539769649506, + "loss_delivery": 0.3752159357070923, + "loss_biodist": 0.19700948819518088, + "loss_toxic": 0.0033008580910973253, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.7878787878787878, + "acc_toxic": 1.0 + }, + { + "loss": 2.0603028774261474, + "loss_size": 0.4996716260910034, + "loss_pdi": 0.5430601418018342, + "loss_ee": 0.623375853896141, + "loss_delivery": 0.24462970793247224, + "loss_biodist": 0.19848335832357406, + "loss_toxic": 0.007659496087580919, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.7272727272727273, + "acc_toxic": 1.0 + }, + { + "loss": 1.9372182607650756, + "loss_size": 0.5049063771963119, + "loss_pdi": 0.5282617956399918, + "loss_ee": 0.6546011686325073, + "loss_delivery": 0.11980208605527878, + "loss_biodist": 0.18347663804888725, + "loss_toxic": 0.004195089940913022, + "acc_pdi": 0.7878787878787878, + "acc_ee": 0.696969696969697, + "acc_toxic": 1.0 + }, + { + "loss": 1.9230464458465577, + "loss_size": 0.4861533761024475, + "loss_pdi": 0.4957872599363327, + "loss_ee": 0.6811205118894577, + "loss_delivery": 0.12559136152267455, + "loss_biodist": 0.19170617461204528, + "loss_toxic": 0.003053023712709546, + "acc_pdi": 0.7878787878787878, + "acc_ee": 0.7272727272727273, + "acc_toxic": 1.0 + }, + { + "loss": 1.9245877981185913, + "loss_size": 0.47571042776107786, + "loss_pdi": 0.5140088766813278, + "loss_ee": 0.6643553763628006, + "loss_delivery": 0.13551929593086243, + "loss_biodist": 0.19236155450344086, + "loss_toxic": 0.00303305524867028, + "acc_pdi": 0.7878787878787878, + "acc_ee": 0.7272727272727273, + "acc_toxic": 1.0 + } + ] +} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_1/splits.json b/models/cv5_sample/baseline/seed42/outer_fold_1/splits.json new file mode 100644 index 0000000..1773fe5 --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_1/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [0, 3, 5, 6, 7, 10, 11, 12, 13, 15, 16, 18, 19, 20, 21, 24, 26, 27, 29, 30, 31, 32, 33, 35, 36, 38, 39, 40, 42, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 65, 66, 67, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 97, 98, 99, 100, 101, 102, 103, 105, 106, 107, 108, 109, 110, 112, 114, 115, 118, 119, 120, 121, 123, 124, 125, 126, 128, 129, 130, 131, 132, 136, 137, 140, 141, 142, 143, 144, 145, 146, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 161, 162, 164, 165, 166, 167, 168, 169, 171, 172, 173, 174, 175, 176, 177, 178, 179, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 194, 195, 196, 197, 198, 199, 201, 202, 204, 205, 206, 207, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 225, 226, 228, 229, 231, 232, 233, 234, 235, 236, 237, 238, 240, 242, 243, 244, 246, 247, 248, 250, 251, 252, 254, 255, 256, 258, 259, 260, 261, 262, 263, 264, 266, 268, 269, 270, 271, 273, 274, 275, 276, 277, 278, 279, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 295, 296, 297, 300, 301, 302, 303, 304, 305, 306, 307, 308, 310, 311, 313, 314, 315, 316, 317, 318, 319, 321, 322, 323, 324, 325, 327, 328, 329, 330, 331, 333, 334, 335, 337, 338, 339, 340, 341, 342, 343, 344, 345, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 361, 362, 364, 365, 366, 367, 368, 369, 373, 374, 375, 376, 378, 379, 380, 381, 382, 384, 387, 388, 389, 391, 392, 393, 394, 396, 397, 398, 399, 402, 403, 404, 405, 406, 408, 409, 410, 411, 412, 413, 414, 416, 417, 419], "outer_test_idx": [1, 2, 4, 8, 9, 14, 17, 22, 23, 25, 28, 34, 37, 41, 43, 45, 63, 64, 68, 85, 96, 104, 111, 113, 116, 117, 122, 127, 133, 134, 135, 138, 139, 147, 160, 163, 170, 180, 191, 192, 193, 200, 203, 208, 224, 227, 230, 239, 241, 245, 249, 253, 257, 265, 267, 272, 280, 294, 298, 299, 309, 312, 320, 326, 332, 336, 346, 347, 360, 363, 370, 371, 372, 377, 383, 385, 386, 390, 395, 400, 401, 407, 415, 418]} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_1/test_metrics.json b/models/cv5_sample/baseline/seed42/outer_fold_1/test_metrics.json new file mode 100644 index 0000000..b91211c --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_1/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.30847479651898757, + "rmse": 0.5554050742647096, + "mae": 0.41110294015102444, + "r2": 0.05802764121606785 + }, + "delivery": { + "n_samples": 61, + "mse": 0.9177343265746907, + "rmse": 0.9579845127008529, + "mae": 0.589223776898179, + "r2": 0.30534127327020133 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.5875, + "recall": 0.5952380952380952, + "f1": 0.5902439024390244 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5833333333333334, + "precision": 0.5230357877416701, + "recall": 0.5150801579373008, + "f1": 0.49626374349858454 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9836065573770492, + "precision": 0.9915254237288136, + "recall": 0.8333333333333333, + "f1": 0.8957264957264958 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.13675394519783235, + "js_divergence": 0.03248669854336822 + } +} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_2/best_params.json b/models/cv5_sample/baseline/seed42/outer_fold_2/best_params.json new file mode 100644 index 0000000..df17e17 --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_2/best_params.json @@ -0,0 +1,16 @@ +{ + "dropout": 0.19323529834582587, + "lr": 0.000980484886752915, + "weight_decay": 0.00014367509113739864, + "backbone_lr_ratio": 0.13093310334961422, + "moe_n_experts": 2, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_2/epoch_mean.json b/models/cv5_sample/baseline/seed42/outer_fold_2/epoch_mean.json new file mode 100644 index 0000000..735a6b9 --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_2/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 11} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_2/history.json b/models/cv5_sample/baseline/seed42/outer_fold_2/history.json new file mode 100644 index 0000000..736f45d --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_2/history.json @@ -0,0 +1,237 @@ +{ + "train": [ + { + "loss": 4.981144982415277, + "loss_size": 0.8744071665647868, + "loss_pdi": 0.6796890819394911, + "loss_ee": 1.0396443009376526, + "loss_delivery": 0.8872663096801655, + "loss_biodist": 0.9659920004574029, + "loss_toxic": 0.5519921908507476 + }, + { + "loss": 3.534698486328125, + "loss_size": 0.502043268769174, + "loss_pdi": 0.6077678847957302, + "loss_ee": 0.9173400402069092, + "loss_delivery": 0.7972146357233459, + "loss_biodist": 0.47308555164852656, + "loss_toxic": 0.26515947262177597 + }, + { + "loss": 3.369716721612054, + "loss_size": 0.8003431667347212, + "loss_pdi": 0.5898223374340985, + "loss_ee": 0.8692510144130604, + "loss_delivery": 0.6483961460558144, + "loss_biodist": 0.33141391241067164, + "loss_toxic": 0.16803750934431683 + }, + { + "loss": 3.485442135785077, + "loss_size": 0.8031057560363332, + "loss_pdi": 0.581869965469515, + "loss_ee": 0.891696730981002, + "loss_delivery": 0.7873844331180727, + "loss_biodist": 0.30878648705579137, + "loss_toxic": 0.1734091161146156 + }, + { + "loss": 3.1509077484543258, + "loss_size": 0.7464953818031259, + "loss_pdi": 0.5425146785942284, + "loss_ee": 0.8121652844789866, + "loss_delivery": 0.7709234127321759, + "loss_biodist": 0.23068079734976227, + "loss_toxic": 0.11076271264008372 + }, + { + "loss": 2.9153835676811837, + "loss_size": 0.7114814908520596, + "loss_pdi": 0.5249752043872267, + "loss_ee": 0.7787095694928556, + "loss_delivery": 0.6547171264081388, + "loss_biodist": 0.21912198936617053, + "loss_toxic": 0.08922003149180799 + }, + { + "loss": 2.7910783709706486, + "loss_size": 0.6730798660097895, + "loss_pdi": 0.48799943561489517, + "loss_ee": 0.7486014172837541, + "loss_delivery": 0.6180728810864526, + "loss_biodist": 0.20473402114333333, + "loss_toxic": 0.11729335879302911 + }, + { + "loss": 2.6673463969617277, + "loss_size": 0.6288768926987777, + "loss_pdi": 0.4827019000375593, + "loss_ee": 0.7593149873050483, + "loss_delivery": 0.5716850237065071, + "loss_biodist": 0.1800537146627903, + "loss_toxic": 0.10778728406876326 + }, + { + "loss": 2.5156604889276863, + "loss_size": 0.5605991873386744, + "loss_pdi": 0.4511222758808652, + "loss_ee": 0.7027070796167528, + "loss_delivery": 0.5257743572866594, + "loss_biodist": 0.18494447743570483, + "loss_toxic": 0.14391961756376298 + }, + { + "loss": 2.589712107503736, + "loss_size": 0.6075560066949677, + "loss_pdi": 0.4832051074182665, + "loss_ee": 0.742237118450371, + "loss_delivery": 0.5646769211098954, + "loss_biodist": 0.20049028380497083, + "loss_toxic": 0.06180886638577323 + }, + { + "loss": 2.605213010633314, + "loss_size": 0.6251636191799834, + "loss_pdi": 0.46899717240720185, + "loss_ee": 0.701141885003528, + "loss_delivery": 0.5535404454614665, + "loss_biodist": 0.163242267193021, + "loss_toxic": 0.14924246485571604 + } + ], + "val": [ + { + "loss": 7.663464832305908, + "loss_size": 3.313545787334442, + "loss_pdi": 0.6900765538215637, + "loss_ee": 1.0373500823974608, + "loss_delivery": 1.899225541949272, + "loss_biodist": 0.5267939090728759, + "loss_toxic": 0.2363927185535431, + "acc_pdi": 0.48484848484848486, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.95 + }, + { + "loss": 7.12261176109314, + "loss_size": 3.4245932430028914, + "loss_pdi": 0.7202669441699981, + "loss_ee": 0.9270447134971619, + "loss_delivery": 1.7875666916370392, + "loss_biodist": 0.239187753200531, + "loss_toxic": 0.07735542804002762, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.95 + }, + { + "loss": 6.7173912525177, + "loss_size": 3.356651684641838, + "loss_pdi": 0.6973654150962829, + "loss_ee": 0.9874879479408264, + "loss_delivery": 1.495903843641281, + "loss_biodist": 0.1953073710203171, + "loss_toxic": 0.04306442430242896, + "acc_pdi": 0.5151515151515151, + "acc_ee": 0.6363636363636364, + "acc_toxic": 0.95 + }, + { + "loss": 6.669378137588501, + "loss_size": 3.177801153063774, + "loss_pdi": 0.6953718304634094, + "loss_ee": 0.9659196972846985, + "loss_delivery": 1.685699325799942, + "loss_biodist": 0.1821014702320099, + "loss_toxic": 0.04902317344676703, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.95 + }, + { + "loss": 6.0091077327728275, + "loss_size": 2.877595055103302, + "loss_pdi": 0.6217953264713287, + "loss_ee": 1.0046721696853638, + "loss_delivery": 1.47297545671463, + "loss_biodist": 0.09909218698740005, + "loss_toxic": 0.02542670485563576, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.5757575757575758, + "acc_toxic": 1.0 + }, + { + "loss": 5.837002372741699, + "loss_size": 2.829563948512077, + "loss_pdi": 0.6526294827461243, + "loss_ee": 0.9968664407730102, + "loss_delivery": 1.3078757762908935, + "loss_biodist": 0.09006003737449646, + "loss_toxic": 0.050850869109854105, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.95 + }, + { + "loss": 5.693418264389038, + "loss_size": 2.7425797268748284, + "loss_pdi": 0.6487438917160034, + "loss_ee": 1.0159194469451904, + "loss_delivery": 1.2614001840353013, + "loss_biodist": 0.08654961585998536, + "loss_toxic": 0.03482890441082418, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.95 + }, + { + "loss": 5.848294258117676, + "loss_size": 2.9319378077983855, + "loss_pdi": 0.6582076787948609, + "loss_ee": 1.0355260014533996, + "loss_delivery": 1.2072753131389617, + "loss_biodist": 0.09763360768556595, + "loss_toxic": 0.015172032522968948, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.5454545454545454, + "acc_toxic": 1.0 + }, + { + "loss": 5.659598445892334, + "loss_size": 2.7996189504861833, + "loss_pdi": 0.650446206331253, + "loss_ee": 0.9975044012069703, + "loss_delivery": 1.197677844762802, + "loss_biodist": 0.0949702337384224, + "loss_toxic": 0.015282840328291058, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.6060606060606061, + "acc_toxic": 1.0 + }, + { + "loss": 5.686920213699341, + "loss_size": 2.7936870113015173, + "loss_pdi": 0.6796894907951355, + "loss_ee": 1.025844794511795, + "loss_delivery": 1.1696001946926118, + "loss_biodist": 0.09400582611560822, + "loss_toxic": 0.023353599617257713, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.5454545454545454, + "acc_toxic": 1.0 + }, + { + "loss": 5.827069425582886, + "loss_size": 2.8803999423980713, + "loss_pdi": 0.6822858631610871, + "loss_ee": 1.0088962197303772, + "loss_delivery": 1.2416288018226624, + "loss_biodist": 0.09613501951098442, + "loss_toxic": 0.017144279484637082, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.5757575757575758, + "acc_toxic": 1.0 + } + ] +} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_2/splits.json b/models/cv5_sample/baseline/seed42/outer_fold_2/splits.json new file mode 100644 index 0000000..14df306 --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_2/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 36, 37, 38, 39, 40, 41, 43, 44, 45, 47, 48, 49, 50, 51, 55, 56, 57, 58, 59, 60, 61, 63, 64, 65, 68, 69, 70, 72, 74, 76, 78, 79, 80, 82, 83, 84, 85, 86, 87, 88, 91, 93, 94, 95, 96, 97, 100, 101, 102, 103, 104, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 142, 143, 145, 147, 148, 149, 150, 151, 152, 153, 156, 157, 158, 159, 160, 161, 162, 163, 165, 166, 167, 168, 170, 171, 172, 173, 174, 175, 177, 179, 180, 181, 183, 184, 185, 186, 187, 188, 191, 192, 193, 194, 196, 197, 199, 200, 202, 203, 204, 205, 207, 208, 209, 210, 211, 212, 213, 214, 215, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 232, 233, 235, 236, 237, 238, 239, 240, 241, 242, 243, 245, 246, 249, 250, 251, 252, 253, 254, 255, 256, 257, 259, 260, 261, 263, 264, 265, 266, 267, 268, 269, 271, 272, 273, 274, 275, 276, 280, 281, 282, 283, 284, 285, 287, 289, 290, 293, 294, 295, 296, 297, 298, 299, 300, 302, 303, 305, 307, 308, 309, 310, 311, 312, 313, 314, 315, 317, 318, 320, 321, 322, 323, 324, 326, 327, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 358, 360, 361, 363, 364, 365, 366, 368, 369, 370, 371, 372, 373, 376, 377, 378, 379, 380, 381, 382, 383, 385, 386, 387, 388, 389, 390, 392, 394, 395, 398, 399, 400, 401, 402, 404, 405, 407, 408, 409, 410, 412, 413, 414, 415, 416, 417, 418], "outer_test_idx": [10, 12, 15, 31, 35, 42, 46, 52, 53, 54, 62, 66, 67, 71, 73, 75, 77, 81, 89, 90, 92, 98, 99, 105, 123, 124, 141, 144, 146, 154, 155, 164, 169, 176, 178, 182, 189, 190, 195, 198, 201, 206, 216, 217, 218, 219, 231, 234, 244, 247, 248, 258, 262, 270, 277, 278, 279, 286, 288, 291, 292, 301, 304, 306, 316, 319, 325, 328, 343, 357, 359, 362, 367, 374, 375, 384, 391, 393, 396, 397, 403, 406, 411, 419]} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_2/test_metrics.json b/models/cv5_sample/baseline/seed42/outer_fold_2/test_metrics.json new file mode 100644 index 0000000..0430c7c --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_2/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.3450456742192763, + "rmse": 0.5874058854142307, + "mae": 0.46413040039173903, + "r2": 0.31854095833662033 + }, + "delivery": { + "n_samples": 60, + "mse": 0.633114832363136, + "rmse": 0.7956851339337289, + "mae": 0.5481160653211797, + "r2": 0.15721442419210252 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7142857142857143, + "precision": 0.6517857142857143, + "recall": 0.6744868035190617, + "f1": 0.6583050847457628 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6428571428571429, + "precision": 0.5853276353276353, + "recall": 0.6349206349206349, + "f1": 0.5970747928194737 + }, + "toxic": { + "n_samples": 61, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.1542080131383728, + "js_divergence": 0.03808539778255157 + } +} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_3/best_params.json b/models/cv5_sample/baseline/seed42/outer_fold_3/best_params.json new file mode 100644 index 0000000..df17e17 --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_3/best_params.json @@ -0,0 +1,16 @@ +{ + "dropout": 0.19323529834582587, + "lr": 0.000980484886752915, + "weight_decay": 0.00014367509113739864, + "backbone_lr_ratio": 0.13093310334961422, + "moe_n_experts": 2, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_3/epoch_mean.json b/models/cv5_sample/baseline/seed42/outer_fold_3/epoch_mean.json new file mode 100644 index 0000000..735a6b9 --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_3/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 11} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_3/history.json b/models/cv5_sample/baseline/seed42/outer_fold_3/history.json new file mode 100644 index 0000000..dadd934 --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_3/history.json @@ -0,0 +1,237 @@ +{ + "train": [ + { + "loss": 4.500459426158184, + "loss_size": 0.5179172538824983, + "loss_pdi": 0.6786781702492688, + "loss_ee": 1.0435632644472896, + "loss_delivery": 0.8494648298097623, + "loss_biodist": 0.9263661117167086, + "loss_toxic": 0.5011754672269564 + }, + { + "loss": 3.5351730681754447, + "loss_size": 0.4758393716167759, + "loss_pdi": 0.6050793609103641, + "loss_ee": 0.9199129455798382, + "loss_delivery": 0.8140221713362513, + "loss_biodist": 0.46644948382635376, + "loss_toxic": 0.28111603362737475 + }, + { + "loss": 3.054368998553302, + "loss_size": 0.4495431559311377, + "loss_pdi": 0.5573401378618704, + "loss_ee": 0.8661556098912213, + "loss_delivery": 0.7387672813760268, + "loss_biodist": 0.3471993232095564, + "loss_toxic": 0.13824860089634722 + }, + { + "loss": 3.0508435287991085, + "loss_size": 0.44718711551379514, + "loss_pdi": 0.5715750842481047, + "loss_ee": 0.8271728351309493, + "loss_delivery": 0.8101292844559695, + "loss_biodist": 0.30609360800401586, + "loss_toxic": 0.14899400996698722 + }, + { + "loss": 2.663577508282017, + "loss_size": 0.41014224129754145, + "loss_pdi": 0.5433303649361069, + "loss_ee": 0.7740156594160441, + "loss_delivery": 0.6729272297910742, + "loss_biodist": 0.21821164641831373, + "loss_toxic": 0.10631358106872318 + }, + { + "loss": 2.557415685138187, + "loss_size": 0.4118609937864381, + "loss_pdi": 0.5263543410881145, + "loss_ee": 0.7567270873366175, + "loss_delivery": 0.6311398055907842, + "loss_biodist": 0.2105274689761368, + "loss_toxic": 0.0864857490224814 + }, + { + "loss": 2.3681529148204907, + "loss_size": 0.34351927102417557, + "loss_pdi": 0.4974673124583992, + "loss_ee": 0.7299669586323403, + "loss_delivery": 0.5682296187088296, + "loss_biodist": 0.18714645907685562, + "loss_toxic": 0.10800810708865724 + }, + { + "loss": 2.3271716382052445, + "loss_size": 0.3356305380930772, + "loss_pdi": 0.4922029255209742, + "loss_ee": 0.7045397452405981, + "loss_delivery": 0.6236170645098429, + "loss_biodist": 0.1928157155980935, + "loss_toxic": 0.05377595526845874 + }, + { + "loss": 2.3417651943258337, + "loss_size": 0.33690569489388855, + "loss_pdi": 0.49544163652368495, + "loss_ee": 0.7062375666321935, + "loss_delivery": 0.6490140618706072, + "loss_biodist": 0.17358206846826785, + "loss_toxic": 0.060061405567024405 + }, + { + "loss": 2.245423777683361, + "loss_size": 0.35165876744164004, + "loss_pdi": 0.48304893237513463, + "loss_ee": 0.635595595111718, + "loss_delivery": 0.5934803685223734, + "loss_biodist": 0.1646844497000849, + "loss_toxic": 0.08570577780640609 + }, + { + "loss": 2.2503104499868445, + "loss_size": 0.3315257770386902, + "loss_pdi": 0.4818291132514541, + "loss_ee": 0.7003061497533644, + "loss_delivery": 0.57664611291241, + "loss_biodist": 0.16049769621443105, + "loss_toxic": 0.07698796648686324 + } + ], + "val": [ + { + "loss": 5.7637759923934935, + "loss_size": 2.7772069811820983, + "loss_pdi": 0.6040945172309875, + "loss_ee": 0.8834547519683837, + "loss_delivery": 0.8618888959288598, + "loss_biodist": 0.40433400869369507, + "loss_toxic": 0.22842871248722077, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.7272727272727273, + "acc_toxic": 0.782608695652174 + }, + { + "loss": 5.360143804550171, + "loss_size": 2.749151438474655, + "loss_pdi": 0.5780697166919708, + "loss_ee": 0.7454802870750428, + "loss_delivery": 0.8378195464611053, + "loss_biodist": 0.2229689359664917, + "loss_toxic": 0.21747921109199525, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.6666666666666666, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 5.201284193992615, + "loss_size": 2.6817896857857706, + "loss_pdi": 0.6185303628444672, + "loss_ee": 0.7450856685638427, + "loss_delivery": 0.773091234266758, + "loss_biodist": 0.20598501861095428, + "loss_toxic": 0.16629037708044053, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.6363636363636364, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 4.911232304573059, + "loss_size": 2.483246049284935, + "loss_pdi": 0.5181228935718536, + "loss_ee": 0.8096402883529663, + "loss_delivery": 0.7650228798389435, + "loss_biodist": 0.17875694036483764, + "loss_toxic": 0.14957769010216, + "acc_pdi": 0.8181818181818182, + "acc_ee": 0.6363636363636364, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 5.086032152175903, + "loss_size": 2.546382114291191, + "loss_pdi": 0.5509662330150604, + "loss_ee": 0.8134036421775818, + "loss_delivery": 0.778993546962738, + "loss_biodist": 0.16294443011283874, + "loss_toxic": 0.21995164193212985, + "acc_pdi": 0.8181818181818182, + "acc_ee": 0.7272727272727273, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 5.123322796821594, + "loss_size": 2.5980996996164323, + "loss_pdi": 0.587708693742752, + "loss_ee": 0.7929542601108551, + "loss_delivery": 0.8301725268363953, + "loss_biodist": 0.1425508365035057, + "loss_toxic": 0.1622565558180213, + "acc_pdi": 0.8181818181818182, + "acc_ee": 0.6666666666666666, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 4.9730717420578, + "loss_size": 2.5694424003362655, + "loss_pdi": 0.5331394076347351, + "loss_ee": 0.8161154568195343, + "loss_delivery": 0.771108528971672, + "loss_biodist": 0.16225484609603882, + "loss_toxic": 0.11304349759593606, + "acc_pdi": 0.8181818181818182, + "acc_ee": 0.696969696969697, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 4.881006336212158, + "loss_size": 2.4189414471387862, + "loss_pdi": 0.5496642231941223, + "loss_ee": 0.8132100462913513, + "loss_delivery": 0.8486790001392365, + "loss_biodist": 0.15106511116027832, + "loss_toxic": 0.10017689382657409, + "acc_pdi": 0.8181818181818182, + "acc_ee": 0.6666666666666666, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 4.704700064659119, + "loss_size": 2.282734674215317, + "loss_pdi": 0.5450537085533143, + "loss_ee": 0.8220642626285553, + "loss_delivery": 0.7426864296197891, + "loss_biodist": 0.15404343008995056, + "loss_toxic": 0.15513503681868315, + "acc_pdi": 0.8181818181818182, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 4.758828639984131, + "loss_size": 2.3555777221918106, + "loss_pdi": 0.5239109575748444, + "loss_ee": 0.8412651300430298, + "loss_delivery": 0.7657725661993027, + "loss_biodist": 0.13986787497997283, + "loss_toxic": 0.13041157741099596, + "acc_pdi": 0.8181818181818182, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 4.7822669506072994, + "loss_size": 2.370918756723404, + "loss_pdi": 0.5238461703062057, + "loss_ee": 0.8227214813232422, + "loss_delivery": 0.77011858522892, + "loss_biodist": 0.14329722970724107, + "loss_toxic": 0.1449897738173604, + "acc_pdi": 0.8181818181818182, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.8695652173913043 + } + ] +} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_3/splits.json b/models/cv5_sample/baseline/seed42/outer_fold_3/splits.json new file mode 100644 index 0000000..7a2b13e --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_3/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [0, 1, 2, 4, 5, 7, 8, 9, 10, 11, 12, 14, 15, 16, 17, 20, 21, 22, 23, 24, 25, 27, 28, 29, 30, 31, 32, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 45, 46, 47, 51, 52, 53, 54, 55, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 73, 74, 75, 76, 77, 78, 79, 81, 82, 83, 84, 85, 86, 87, 89, 90, 91, 92, 93, 95, 96, 97, 98, 99, 100, 102, 104, 105, 106, 109, 110, 111, 112, 113, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 127, 129, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 146, 147, 148, 150, 151, 152, 153, 154, 155, 156, 158, 159, 160, 161, 162, 163, 164, 165, 166, 168, 169, 170, 172, 173, 174, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 189, 190, 191, 192, 193, 194, 195, 198, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 212, 213, 214, 215, 216, 217, 218, 219, 221, 223, 224, 226, 227, 228, 230, 231, 233, 234, 235, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 251, 252, 253, 254, 255, 256, 257, 258, 259, 262, 263, 264, 265, 266, 267, 268, 269, 270, 272, 273, 274, 275, 276, 277, 278, 279, 280, 282, 284, 286, 288, 289, 290, 291, 292, 293, 294, 295, 297, 298, 299, 301, 302, 304, 305, 306, 307, 308, 309, 310, 312, 314, 315, 316, 317, 319, 320, 321, 325, 326, 327, 328, 331, 332, 334, 336, 340, 341, 343, 344, 346, 347, 348, 350, 352, 353, 357, 359, 360, 361, 362, 363, 364, 366, 367, 368, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 382, 383, 384, 385, 386, 387, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 400, 401, 402, 403, 404, 406, 407, 410, 411, 412, 414, 415, 416, 418, 419], "outer_test_idx": [3, 6, 13, 18, 19, 26, 33, 44, 48, 49, 50, 56, 57, 72, 80, 88, 94, 101, 103, 107, 108, 114, 126, 128, 130, 145, 149, 157, 167, 171, 175, 188, 196, 197, 199, 211, 220, 222, 225, 229, 232, 236, 250, 260, 261, 271, 281, 283, 285, 287, 296, 300, 303, 311, 313, 318, 322, 323, 324, 329, 330, 333, 335, 337, 338, 339, 342, 345, 349, 351, 354, 355, 356, 358, 365, 369, 381, 388, 399, 405, 408, 409, 413, 417]} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_3/test_metrics.json b/models/cv5_sample/baseline/seed42/outer_fold_3/test_metrics.json new file mode 100644 index 0000000..b1fe7af --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_3/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 83, + "mse": 1.763092308495774, + "rmse": 1.3278148622815509, + "mae": 0.5504888193233006, + "r2": 0.0027815521929155373 + }, + "delivery": { + "n_samples": 59, + "mse": 0.6925828062408207, + "rmse": 0.8322156008155703, + "mae": 0.5598128846357182, + "r2": 0.32672722594328685 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7380952380952381, + "precision": 0.6612903225806452, + "recall": 0.6612903225806452, + "f1": 0.6612903225806452 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5952380952380952, + "precision": 0.5301311591634171, + "recall": 0.5438914027149321, + "f1": 0.5247639796659405 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9333333333333333, + "precision": 0.7142857142857143, + "recall": 0.9649122807017544, + "f1": 0.7818181818181817 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.15787098167920866, + "js_divergence": 0.04007834983720861 + } +} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_4/best_params.json b/models/cv5_sample/baseline/seed42/outer_fold_4/best_params.json new file mode 100644 index 0000000..df17e17 --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_4/best_params.json @@ -0,0 +1,16 @@ +{ + "dropout": 0.19323529834582587, + "lr": 0.000980484886752915, + "weight_decay": 0.00014367509113739864, + "backbone_lr_ratio": 0.13093310334961422, + "moe_n_experts": 2, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_4/epoch_mean.json b/models/cv5_sample/baseline/seed42/outer_fold_4/epoch_mean.json new file mode 100644 index 0000000..735a6b9 --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_4/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 11} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_4/history.json b/models/cv5_sample/baseline/seed42/outer_fold_4/history.json new file mode 100644 index 0000000..e87ac85 --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_4/history.json @@ -0,0 +1,237 @@ +{ + "train": [ + { + "loss": 5.102151986714956, + "loss_size": 1.1567820576397148, + "loss_pdi": 0.6515501222094974, + "loss_ee": 1.0294628127201184, + "loss_delivery": 0.8426589999045875, + "loss_biodist": 0.9110404494646434, + "loss_toxic": 0.5230041408055538 + }, + { + "loss": 4.055108985385379, + "loss_size": 1.0688643268234022, + "loss_pdi": 0.5984339689886248, + "loss_ee": 0.9147920866270323, + "loss_delivery": 0.798665193689836, + "loss_biodist": 0.4416384439210634, + "loss_toxic": 0.2584042294403991 + }, + { + "loss": 3.8014610168096183, + "loss_size": 1.0324330774916184, + "loss_pdi": 0.5652096682303661, + "loss_ee": 0.9040436511104172, + "loss_delivery": 0.7405963850182455, + "loss_biodist": 0.3845312357754321, + "loss_toxic": 0.21792648017809196 + }, + { + "loss": 3.3098922484629862, + "loss_size": 0.6985325795170423, + "loss_pdi": 0.5925048146698926, + "loss_ee": 0.846102629158948, + "loss_delivery": 0.7961102732129999, + "loss_biodist": 0.2757947507339555, + "loss_toxic": 0.15982103052020474 + }, + { + "loss": 3.176285627725962, + "loss_size": 0.9075968438709104, + "loss_pdi": 0.5314604060069935, + "loss_ee": 0.8254372380875252, + "loss_delivery": 0.6657844270906739, + "loss_biodist": 0.21802176696223183, + "loss_toxic": 0.0905371925390854 + }, + { + "loss": 3.274916900170816, + "loss_size": 0.9856320102069829, + "loss_pdi": 0.5265752342907158, + "loss_ee": 0.7516567006304458, + "loss_delivery": 0.7083130106329918, + "loss_biodist": 0.2212505919506421, + "loss_toxic": 0.1400652697681122 + }, + { + "loss": 2.9465325329754806, + "loss_size": 0.8816265802125673, + "loss_pdi": 0.5074402512730779, + "loss_ee": 0.757787544179607, + "loss_delivery": 0.6106935801940996, + "loss_biodist": 0.19856224893718152, + "loss_toxic": 0.05817761752011909 + }, + { + "loss": 2.957754476650341, + "loss_size": 0.854996056572811, + "loss_pdi": 0.48137593349894964, + "loss_ee": 0.7679745611306783, + "loss_delivery": 0.6376201487272173, + "loss_biodist": 0.17370915422971184, + "loss_toxic": 0.10946399376671007 + }, + { + "loss": 2.7943139656169995, + "loss_size": 0.7861643652054103, + "loss_pdi": 0.4862389399393185, + "loss_ee": 0.6960344652871828, + "loss_delivery": 0.620618630301308, + "loss_biodist": 0.19384471256587957, + "loss_toxic": 0.07682121185762053 + }, + { + "loss": 2.7292510528822205, + "loss_size": 0.736359557187235, + "loss_pdi": 0.5096078466724705, + "loss_ee": 0.7071713265534993, + "loss_delivery": 0.5689382711677132, + "loss_biodist": 0.2055618997562576, + "loss_toxic": 0.07013601859609873 + }, + { + "loss": 2.688862778045036, + "loss_size": 0.7182406856602913, + "loss_pdi": 0.4632230537968713, + "loss_ee": 0.7033636360555082, + "loss_delivery": 0.6424240871156389, + "loss_biodist": 0.17127765292251432, + "loss_toxic": 0.05990550624886276 + } + ], + "val": [ + { + "loss": 3.6000691413879395, + "loss_size": 0.3701790302991867, + "loss_pdi": 0.5664441585540771, + "loss_ee": 0.8832380712032318, + "loss_delivery": 0.8911014348268509, + "loss_biodist": 0.5329773306846619, + "loss_toxic": 0.3749651610851288, + "acc_pdi": 0.7878787878787878, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 3.0433499336242678, + "loss_size": 0.3372409701347351, + "loss_pdi": 0.4964510649442673, + "loss_ee": 0.7701501965522766, + "loss_delivery": 0.8530217319726944, + "loss_biodist": 0.2348724514245987, + "loss_toxic": 0.36663210168480875, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 2.779363822937012, + "loss_size": 0.30666323006153107, + "loss_pdi": 0.4720880776643753, + "loss_ee": 0.7368923991918563, + "loss_delivery": 0.828652124106884, + "loss_biodist": 0.16220578402280808, + "loss_toxic": 0.3004120915196836, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 3.281400537490845, + "loss_size": 0.5272440910339355, + "loss_pdi": 0.5226441502571106, + "loss_ee": 0.8246899724006653, + "loss_delivery": 0.8534515619277954, + "loss_biodist": 0.28649153709411623, + "loss_toxic": 0.31357612535357476, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.9561454892158507, + "loss_size": 0.2726425051689148, + "loss_pdi": 0.45198590606451033, + "loss_ee": 0.7590681105852127, + "loss_delivery": 0.8566071918583475, + "loss_biodist": 0.10773630291223527, + "loss_toxic": 0.5303979313932359, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.8890976190567015, + "loss_size": 0.34611437618732455, + "loss_pdi": 0.4896680176258087, + "loss_ee": 0.7619812674820423, + "loss_delivery": 0.866328160604462, + "loss_biodist": 0.09204872399568557, + "loss_toxic": 0.37474240330047903, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 2.7058202266693114, + "loss_size": 0.3040961235761642, + "loss_pdi": 0.44319443553686144, + "loss_ee": 0.7749164622277022, + "loss_delivery": 0.7846140109002591, + "loss_biodist": 0.09509205557405949, + "loss_toxic": 0.34775193249806763, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.8554219126701357, + "loss_size": 0.24815200865268708, + "loss_pdi": 0.4591174952685833, + "loss_ee": 0.7767300330102443, + "loss_delivery": 0.75640877969563, + "loss_biodist": 0.09491635411977768, + "loss_toxic": 0.5468520291149617, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.8406914949417112, + "loss_size": 0.2561314105987549, + "loss_pdi": 0.46083473414182663, + "loss_ee": 0.7739205928519368, + "loss_delivery": 0.784732368728146, + "loss_biodist": 0.08942811377346516, + "loss_toxic": 0.5080395077820867, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.8140346884727476, + "loss_size": 0.2685915812849998, + "loss_pdi": 0.4499114632606506, + "loss_ee": 0.8004547510296106, + "loss_delivery": 0.8297218790045008, + "loss_biodist": 0.08691173046827316, + "loss_toxic": 0.4234957183711231, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.763965880870819, + "loss_size": 0.2510989636182785, + "loss_pdi": 0.45986512303352356, + "loss_ee": 0.7952149830758571, + "loss_delivery": 0.8199413266032934, + "loss_biodist": 0.0973357006907463, + "loss_toxic": 0.3888999660965055, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.9444444444444444 + } + ] +} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_4/splits.json b/models/cv5_sample/baseline/seed42/outer_fold_4/splits.json new file mode 100644 index 0000000..8dd0299 --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_4/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23, 25, 26, 28, 29, 31, 33, 34, 35, 37, 41, 42, 43, 44, 45, 46, 48, 49, 50, 51, 52, 53, 54, 56, 57, 59, 60, 62, 63, 64, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 77, 78, 79, 80, 81, 82, 83, 84, 85, 87, 88, 89, 90, 91, 92, 93, 94, 96, 97, 98, 99, 101, 102, 103, 104, 105, 107, 108, 110, 111, 112, 113, 114, 116, 117, 119, 120, 121, 122, 123, 124, 126, 127, 128, 130, 132, 133, 134, 135, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 149, 152, 154, 155, 157, 160, 161, 162, 163, 164, 165, 166, 167, 169, 170, 171, 173, 174, 175, 176, 177, 178, 180, 181, 182, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 203, 205, 206, 207, 208, 209, 211, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 229, 230, 231, 232, 234, 235, 236, 238, 239, 240, 241, 243, 244, 245, 246, 247, 248, 249, 250, 251, 253, 254, 257, 258, 260, 261, 262, 263, 264, 265, 266, 267, 270, 271, 272, 273, 277, 278, 279, 280, 281, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 298, 299, 300, 301, 302, 303, 304, 305, 306, 309, 311, 312, 313, 314, 315, 316, 318, 319, 320, 322, 323, 324, 325, 326, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 341, 342, 343, 345, 346, 347, 349, 350, 351, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 365, 367, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 381, 383, 384, 385, 386, 388, 390, 391, 392, 393, 394, 395, 396, 397, 399, 400, 401, 402, 403, 405, 406, 407, 408, 409, 410, 411, 413, 415, 416, 417, 418, 419], "outer_test_idx": [0, 5, 11, 21, 24, 27, 30, 32, 36, 38, 39, 40, 47, 55, 58, 61, 65, 76, 86, 95, 100, 106, 109, 115, 118, 125, 129, 131, 136, 148, 150, 151, 153, 156, 158, 159, 168, 172, 179, 183, 184, 185, 202, 204, 210, 212, 213, 228, 233, 237, 242, 252, 255, 256, 259, 268, 269, 274, 275, 276, 282, 297, 307, 308, 310, 317, 321, 327, 340, 344, 348, 352, 364, 366, 368, 379, 380, 382, 387, 389, 398, 404, 412, 414]} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/outer_fold_4/test_metrics.json b/models/cv5_sample/baseline/seed42/outer_fold_4/test_metrics.json new file mode 100644 index 0000000..3068d82 --- /dev/null +++ b/models/cv5_sample/baseline/seed42/outer_fold_4/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.5697698999186622, + "rmse": 0.75483104064331, + "mae": 0.45241403512634515, + "r2": 0.23757709942203908 + }, + "delivery": { + "n_samples": 58, + "mse": 0.8494501546811344, + "rmse": 0.9216562019978677, + "mae": 0.6002610280852894, + "r2": 0.17859572323309747 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7380952380952381, + "precision": 0.6724137931034483, + "recall": 0.6906158357771262, + "f1": 0.6791666666666667 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6324786324786325, + "recall": 0.6799145299145298, + "f1": 0.6406077872744539 + }, + "toxic": { + "n_samples": 59, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.17354171533022728, + "js_divergence": 0.046613494801719065 + } +} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/strata_info.json b/models/cv5_sample/baseline/seed42/strata_info.json new file mode 100644 index 0000000..ef3be58 --- /dev/null +++ b/models/cv5_sample/baseline/seed42/strata_info.json @@ -0,0 +1,60 @@ +{ + "original_strata_counts": { + "T0|P0|E0": "5", + "T0|P0|E1": "58", + "T0|P0|E2": "169", + "T0|P1|E0": "1", + "T0|P1|E1": "17", + "T0|P1|E2": "32", + "T0|P2|E2": "3", + "T1|P0|E2": "9", + "T1|P1|E2": "5", + "TNA|P0|E0": "28", + "TNA|P0|E1": "21", + "TNA|P0|E2": "20", + "TNA|P1|E0": "29", + "TNA|P1|E1": "7", + "TNA|P1|E2": "14", + "TNA|P2|E2": "1", + "TNA|P3|E0": "1" + }, + "rare_strata": [ + "T0|P1|E0", + "T0|P2|E2", + "TNA|P2|E2", + "TNA|P3|E0" + ], + "final_strata": [ + "RARE", + "T0|P0|E0", + "T0|P0|E1", + "T0|P0|E2", + "T0|P1|E1", + "T0|P1|E2", + "T1|P0|E2", + "T1|P1|E2", + "TNA|P0|E0", + "TNA|P0|E1", + "TNA|P0|E2", + "TNA|P1|E0", + "TNA|P1|E1", + "TNA|P1|E2" + ], + "final_strata_counts": { + "RARE": "6", + "T0|P0|E0": "5", + "T0|P0|E1": "58", + "T0|P0|E2": "169", + "T0|P1|E1": "17", + "T0|P1|E2": "32", + "T1|P0|E2": "9", + "T1|P1|E2": "5", + "TNA|P0|E0": "28", + "TNA|P0|E1": "21", + "TNA|P0|E2": "20", + "TNA|P1|E0": "29", + "TNA|P1|E1": "7", + "TNA|P1|E2": "14" + }, + "n_rare_merged": "6" +} \ No newline at end of file diff --git a/models/cv5_sample/baseline/seed42/summary.json b/models/cv5_sample/baseline/seed42/summary.json new file mode 100644 index 0000000..dfe77e7 --- /dev/null +++ b/models/cv5_sample/baseline/seed42/summary.json @@ -0,0 +1,372 @@ +{ + "fold_results": [ + { + "fold": 0, + "best_params": { + "dropout": 0.19323529834582587, + "lr": 0.000980484886752915, + "weight_decay": 0.00014367509113739864, + "backbone_lr_ratio": 0.13093310334961422, + "moe_n_experts": 2, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 11, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.5968842484584136, + "rmse": 1.263678854954222, + "mae": 0.5682635198246284, + "r2": 0.022667532793975265 + }, + "delivery": { + "n_samples": 58, + "mse": 0.5483822645024997, + "rmse": 0.740528368465719, + "mae": 0.5250802371843621, + "r2": 0.3029730502833111 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7142857142857143, + "precision": 0.6737967914438503, + "recall": 0.7084818246614397, + "f1": 0.6785714285714286 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7380952380952381, + "precision": 0.7073879551820729, + "recall": 0.7647619047619049, + "f1": 0.7124896608767578 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9827586206896551, + "precision": 0.9912280701754386, + "recall": 0.75, + "f1": 0.8289085545722714 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.1971223395298808, + "js_divergence": 0.043043902163233595 + } + } + }, + { + "fold": 1, + "best_params": { + "dropout": 0.19323529834582587, + "lr": 0.000980484886752915, + "weight_decay": 0.00014367509113739864, + "backbone_lr_ratio": 0.13093310334961422, + "moe_n_experts": 2, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 11, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.30847479651898757, + "rmse": 0.5554050742647096, + "mae": 0.41110294015102444, + "r2": 0.05802764121606785 + }, + "delivery": { + "n_samples": 61, + "mse": 0.9177343265746907, + "rmse": 0.9579845127008529, + "mae": 0.589223776898179, + "r2": 0.30534127327020133 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.5875, + "recall": 0.5952380952380952, + "f1": 0.5902439024390244 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5833333333333334, + "precision": 0.5230357877416701, + "recall": 0.5150801579373008, + "f1": 0.49626374349858454 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9836065573770492, + "precision": 0.9915254237288136, + "recall": 0.8333333333333333, + "f1": 0.8957264957264958 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.13675394519783235, + "js_divergence": 0.03248669854336822 + } + } + }, + { + "fold": 2, + "best_params": { + "dropout": 0.19323529834582587, + "lr": 0.000980484886752915, + "weight_decay": 0.00014367509113739864, + "backbone_lr_ratio": 0.13093310334961422, + "moe_n_experts": 2, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 11, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.3450456742192763, + "rmse": 0.5874058854142307, + "mae": 0.46413040039173903, + "r2": 0.31854095833662033 + }, + "delivery": { + "n_samples": 60, + "mse": 0.633114832363136, + "rmse": 0.7956851339337289, + "mae": 0.5481160653211797, + "r2": 0.15721442419210252 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7142857142857143, + "precision": 0.6517857142857143, + "recall": 0.6744868035190617, + "f1": 0.6583050847457628 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6428571428571429, + "precision": 0.5853276353276353, + "recall": 0.6349206349206349, + "f1": 0.5970747928194737 + }, + "toxic": { + "n_samples": 61, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.1542080131383728, + "js_divergence": 0.03808539778255157 + } + } + }, + { + "fold": 3, + "best_params": { + "dropout": 0.19323529834582587, + "lr": 0.000980484886752915, + "weight_decay": 0.00014367509113739864, + "backbone_lr_ratio": 0.13093310334961422, + "moe_n_experts": 2, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 11, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.763092308495774, + "rmse": 1.3278148622815509, + "mae": 0.5504888193233006, + "r2": 0.0027815521929155373 + }, + "delivery": { + "n_samples": 59, + "mse": 0.6925828062408207, + "rmse": 0.8322156008155703, + "mae": 0.5598128846357182, + "r2": 0.32672722594328685 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7380952380952381, + "precision": 0.6612903225806452, + "recall": 0.6612903225806452, + "f1": 0.6612903225806452 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5952380952380952, + "precision": 0.5301311591634171, + "recall": 0.5438914027149321, + "f1": 0.5247639796659405 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9333333333333333, + "precision": 0.7142857142857143, + "recall": 0.9649122807017544, + "f1": 0.7818181818181817 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.15787098167920866, + "js_divergence": 0.04007834983720861 + } + } + }, + { + "fold": 4, + "best_params": { + "dropout": 0.19323529834582587, + "lr": 0.000980484886752915, + "weight_decay": 0.00014367509113739864, + "backbone_lr_ratio": 0.13093310334961422, + "moe_n_experts": 2, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 11, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.5697698999186622, + "rmse": 0.75483104064331, + "mae": 0.45241403512634515, + "r2": 0.23757709942203908 + }, + "delivery": { + "n_samples": 58, + "mse": 0.8494501546811344, + "rmse": 0.9216562019978677, + "mae": 0.6002610280852894, + "r2": 0.17859572323309747 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7380952380952381, + "precision": 0.6724137931034483, + "recall": 0.6906158357771262, + "f1": 0.6791666666666667 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6785714285714286, + "precision": 0.6324786324786325, + "recall": 0.6799145299145298, + "f1": 0.6406077872744539 + }, + "toxic": { + "n_samples": 59, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.17354171533022728, + "js_divergence": 0.046613494801719065 + } + } + } + ], + "summary_stats": { + "size": { + "mse_mean": 0.9166533855222226, + "mse_std": 0.6318413727420791, + "rmse_mean": 0.8978271435116048, + "rmse_std": 0.33250534716905716, + "mae_mean": 0.4892799429634075, + "mae_std": 0.06014677111861511, + "r2_mean": 0.12791895679232362, + "r2_std": 0.1264782068138924 + }, + "delivery": { + "mse_mean": 0.7282528768724562, + "mse_std": 0.13657894316164607, + "rmse_mean": 0.8496139635827478, + "rmse_std": 0.08005616626887421, + "mae_mean": 0.5644987984249458, + "mae_std": 0.027329238310088152, + "r2_mean": 0.2541703393843998, + "r2_std": 0.07124134045642266 + }, + "pdi": { + "accuracy_mean": 0.7166666666666667, + "accuracy_std": 0.021821789023599242, + "precision_mean": 0.6493573242827317, + "precision_std": 0.03194842845400596, + "recall_mean": 0.6660225763552736, + "recall_std": 0.038761053417086286, + "f1_mean": 0.6535154810007054, + "f1_std": 0.03277945989198496 + }, + "ee": { + "accuracy_mean": 0.6476190476190476, + "accuracy_std": 0.05664465358747772, + "precision_mean": 0.5956722339786855, + "precision_std": 0.06857668680486774, + "recall_mean": 0.6277137260498604, + "recall_std": 0.09085325454039572, + "f1_mean": 0.5942399928270421, + "f1_std": 0.07818855962865172 + }, + "toxic": { + "accuracy_mean": 0.9799397022800076, + "accuracy_std": 0.02448827002026046, + "precision_mean": 0.9394078416379934, + "precision_std": 0.11262714591271072, + "recall_mean": 0.9096491228070175, + "recall_std": 0.10069013931425785, + "f1_mean": 0.9012906464233896, + "f1_std": 0.0883525548692155 + }, + "biodist": { + "kl_divergence_mean": 0.16389939897510436, + "kl_divergence_std": 0.020316882039832528, + "js_divergence_mean": 0.0400615686256162, + "js_divergence_std": 0.0047561799696747425 + } + } +} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_0/best_params.json b/models/cv5_sample/moe_llm/seed42/outer_fold_0/best_params.json new file mode 100644 index 0000000..7395ce5 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_0/best_params.json @@ -0,0 +1,16 @@ +{ + "dropout": 0.19323529834582587, + "lr": 0.000980484886752915, + "weight_decay": 0.00014367509113739864, + "backbone_lr_ratio": 0.13093310334961422, + "moe_n_experts": 4, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_0/epoch_mean.json b/models/cv5_sample/moe_llm/seed42/outer_fold_0/epoch_mean.json new file mode 100644 index 0000000..735a6b9 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_0/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 11} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_0/history.json b/models/cv5_sample/moe_llm/seed42/outer_fold_0/history.json new file mode 100644 index 0000000..79b49f2 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_0/history.json @@ -0,0 +1,271 @@ +{ + "train": [ + { + "loss": 5.9101777463345915, + "loss_size": 0.8219048421125155, + "loss_pdi": 0.6955033202429075, + "loss_ee": 1.060509283800383, + "loss_delivery": 1.1355218297323666, + "loss_biodist": 0.907981518958066, + "loss_toxic": 0.5266113526917793, + "loss_moe_lb": 2.0565945038924345, + "loss_aux_moe": 1.1926589547782331, + "loss_aux_llm": 1.3497599073358484 + }, + { + "loss": 4.279001815898998, + "loss_size": 0.5204920379092565, + "loss_pdi": 0.643413982681326, + "loss_ee": 0.9418714610305993, + "loss_delivery": 0.8884327468034383, + "loss_biodist": 0.43965512856438355, + "loss_toxic": 0.3611150317498156, + "loss_moe_lb": 2.0773819910513387, + "loss_aux_moe": 0.9263223323064882, + "loss_aux_llm": 1.095245057666624 + }, + { + "loss": 4.1005129363085775, + "loss_size": 0.8035221945595097, + "loss_pdi": 0.6013445999171283, + "loss_ee": 0.8792690650836842, + "loss_delivery": 0.9753083366951024, + "loss_biodist": 0.260869637333058, + "loss_toxic": 0.19401404719699072, + "loss_moe_lb": 2.0695742400916846, + "loss_aux_moe": 0.9956301312792946, + "loss_aux_llm": 1.1593363244791288 + }, + { + "loss": 3.8045457988171965, + "loss_size": 0.7318794419233864, + "loss_pdi": 0.5873883729045456, + "loss_ee": 0.8806033150569813, + "loss_delivery": 0.8927270299880892, + "loss_biodist": 0.24839287150550532, + "loss_toxic": 0.2283072332486611, + "loss_moe_lb": 2.1437587512506022, + "loss_aux_moe": 0.9384613774112753, + "loss_aux_llm": 1.0804562792181969 + }, + { + "loss": 3.240251125516118, + "loss_size": 0.6515310636243304, + "loss_pdi": 0.538264666860168, + "loss_ee": 0.8188068826456327, + "loss_delivery": 0.7787331332628792, + "loss_biodist": 0.17340928879944054, + "loss_toxic": 0.17186429866312727, + "loss_moe_lb": 2.1789169665929435, + "loss_aux_moe": 0.7798387190780124, + "loss_aux_llm": 1.0161442956006206 + }, + { + "loss": 2.756747068585576, + "loss_size": 0.6201924379612949, + "loss_pdi": 0.5108480352807689, + "loss_ee": 0.7891972757674552, + "loss_delivery": 0.5784409728807371, + "loss_biodist": 0.1625980019972131, + "loss_toxic": 0.09638489400842101, + "loss_moe_lb": 2.1672985134898006, + "loss_aux_moe": 0.6525389511842985, + "loss_aux_llm": 0.866097974414761 + }, + { + "loss": 2.8097413456117786, + "loss_size": 0.6104585203165943, + "loss_pdi": 0.49535534148280685, + "loss_ee": 0.7758352208781887, + "loss_delivery": 0.6814873780853845, + "loss_biodist": 0.15574933598573143, + "loss_toxic": 0.13788988649203265, + "loss_moe_lb": 2.149112511325527 + }, + { + "loss": 2.5547807119988106, + "loss_size": 0.49592660246668635, + "loss_pdi": 0.4421862122174856, + "loss_ee": 0.7361203488465902, + "loss_delivery": 0.6629904395825154, + "loss_biodist": 0.14514514481699145, + "loss_toxic": 0.12097700466590657, + "loss_moe_lb": 2.127687138480109 + }, + { + "loss": 2.496382748758471, + "loss_size": 0.49584831820951925, + "loss_pdi": 0.43293014209012726, + "loss_ee": 0.7089789686976252, + "loss_delivery": 0.7198760602119807, + "loss_biodist": 0.1322829470642515, + "loss_toxic": 0.06398353036304281, + "loss_moe_lb": 2.137723809963948 + }, + { + "loss": 2.3917723153088546, + "loss_size": 0.5447976748282845, + "loss_pdi": 0.47060893274642324, + "loss_ee": 0.6911834728073429, + "loss_delivery": 0.5664171859420635, + "loss_biodist": 0.12129703466151212, + "loss_toxic": 0.05351947958272806, + "loss_moe_lb": 2.1298333728635632 + }, + { + "loss": 2.1267837833713843, + "loss_size": 0.2846768913236824, + "loss_pdi": 0.44730447313270055, + "loss_ee": 0.6802649554368612, + "loss_delivery": 0.6072367915027851, + "loss_biodist": 0.12430884417247128, + "loss_toxic": 0.04373953997698688, + "loss_moe_lb": 2.14098026623597 + } + ], + "val": [ + { + "loss": 4.0116489887237545, + "loss_size": 0.4896596247330308, + "loss_pdi": 0.7048518657684326, + "loss_ee": 0.9747519016265869, + "loss_delivery": 0.7728810399770737, + "loss_biodist": 0.706810986995697, + "loss_toxic": 0.37354564368724824, + "loss_moe_lb": 2.0459502696990968, + "acc_pdi": 0.6363636363636364, + "acc_ee": 0.45454545454545453, + "acc_toxic": 0.9 + }, + { + "loss": 3.3040961265563964, + "loss_size": 0.445052208006382, + "loss_pdi": 0.7240784287452697, + "loss_ee": 0.8502145409584045, + "loss_delivery": 0.6241754226386547, + "loss_biodist": 0.35449219942092897, + "loss_toxic": 0.3185396656394005, + "loss_moe_lb": 2.1541085243225098, + "acc_pdi": 0.6363636363636364, + "acc_ee": 0.48484848484848486, + "acc_toxic": 0.9 + }, + { + "loss": 3.4476874828338624, + "loss_size": 0.4868860960006714, + "loss_pdi": 0.7957421660423278, + "loss_ee": 0.8902200803160667, + "loss_delivery": 0.712676465883851, + "loss_biodist": 0.18424639031291007, + "loss_toxic": 0.4164220209233463, + "loss_moe_lb": 2.1585488319396973, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.48484848484848486, + "acc_toxic": 0.9 + }, + { + "loss": 3.6464715957641602, + "loss_size": 0.40821063220500947, + "loss_pdi": 0.9250725567340851, + "loss_ee": 0.8542257234454155, + "loss_delivery": 0.7186837211251259, + "loss_biodist": 0.1707709789276123, + "loss_toxic": 0.6113252578303218, + "loss_moe_lb": 2.218433880805969, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.5151515151515151, + "acc_toxic": 0.9 + }, + { + "loss": 2.98254132270813, + "loss_size": 0.3915577203035355, + "loss_pdi": 0.9889228403568268, + "loss_ee": 0.8030516915023327, + "loss_delivery": 0.6232511460781097, + "loss_biodist": 0.18688103817403318, + "loss_toxic": 0.06858860440552235, + "loss_moe_lb": 2.2559383392333983, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.45454545454545453, + "acc_toxic": 1.0 + }, + { + "loss": 3.17673282623291, + "loss_size": 0.45360138416290285, + "loss_pdi": 1.150715070962906, + "loss_ee": 0.8275567952543497, + "loss_delivery": 0.613179263472557, + "loss_biodist": 0.1491141177713871, + "loss_toxic": 0.08308818615041673, + "loss_moe_lb": 2.248614454269409, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.45454545454545453, + "acc_toxic": 1.0 + }, + { + "loss": 3.5197216987609865, + "loss_size": 0.44484867453575133, + "loss_pdi": 1.3462523818016052, + "loss_ee": 0.8313753753900528, + "loss_delivery": 0.6039341509342193, + "loss_biodist": 0.14939168095588684, + "loss_toxic": 0.24860251196660102, + "loss_moe_lb": 2.2827333211898804, + "acc_pdi": 0.5454545454545454, + "acc_ee": 0.45454545454545453, + "acc_toxic": 0.9 + }, + { + "loss": 2.7240954875946044, + "loss_size": 0.4011200428009033, + "loss_pdi": 0.9446926414966583, + "loss_ee": 0.8310217900201678, + "loss_delivery": 0.4591434508562088, + "loss_biodist": 0.14846671670675277, + "loss_toxic": 0.03622314129024744, + "loss_moe_lb": 2.3130435943603516, + "acc_pdi": 0.6363636363636364, + "acc_ee": 0.5454545454545454, + "acc_toxic": 1.0 + }, + { + "loss": 3.0855964183807374, + "loss_size": 0.4515984505414963, + "loss_pdi": 1.174130117893219, + "loss_ee": 0.8735770538449288, + "loss_delivery": 0.500551962852478, + "loss_biodist": 0.12562411203980445, + "loss_toxic": 0.07660643630661071, + "loss_moe_lb": 2.2815173625946046, + "acc_pdi": 0.5757575757575758, + "acc_ee": 0.48484848484848486, + "acc_toxic": 1.0 + }, + { + "loss": 2.9979569911956787, + "loss_size": 0.4335483551025391, + "loss_pdi": 1.2359270095825194, + "loss_ee": 0.7991622228175401, + "loss_delivery": 0.481366953253746, + "loss_biodist": 0.12457970529794693, + "loss_toxic": 0.04048144668340683, + "loss_moe_lb": 2.2985286951065063, + "acc_pdi": 0.5757575757575758, + "acc_ee": 0.5454545454545454, + "acc_toxic": 1.0 + }, + { + "loss": 3.0956326007843016, + "loss_size": 0.4739636182785034, + "loss_pdi": 1.278123664855957, + "loss_ee": 0.8378281531855464, + "loss_delivery": 0.46590901277959346, + "loss_biodist": 0.12750579938292503, + "loss_toxic": 0.03617289457470178, + "loss_moe_lb": 2.2634385585784913, + "acc_pdi": 0.5757575757575758, + "acc_ee": 0.48484848484848486, + "acc_toxic": 1.0 + } + ] +} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_0/splits.json b/models/cv5_sample/moe_llm/seed42/outer_fold_0/splits.json new file mode 100644 index 0000000..cccb9d8 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_0/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 52, 53, 54, 55, 56, 57, 58, 61, 62, 63, 64, 65, 66, 67, 68, 71, 72, 73, 75, 76, 77, 80, 81, 85, 86, 88, 89, 90, 92, 94, 95, 96, 98, 99, 100, 101, 103, 104, 105, 106, 107, 108, 109, 111, 113, 114, 115, 116, 117, 118, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 133, 134, 135, 136, 138, 139, 141, 144, 145, 146, 147, 148, 149, 150, 151, 153, 154, 155, 156, 157, 158, 159, 160, 163, 164, 167, 168, 169, 170, 171, 172, 175, 176, 178, 179, 180, 182, 183, 184, 185, 188, 189, 190, 191, 192, 193, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 206, 208, 210, 211, 212, 213, 216, 217, 218, 219, 220, 222, 224, 225, 227, 228, 229, 230, 231, 232, 233, 234, 236, 237, 239, 241, 242, 244, 245, 247, 248, 249, 250, 252, 253, 255, 256, 257, 258, 259, 260, 261, 262, 265, 267, 268, 269, 270, 271, 272, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 285, 286, 287, 288, 291, 292, 294, 296, 297, 298, 299, 300, 301, 303, 304, 306, 307, 308, 309, 310, 311, 312, 313, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 332, 333, 335, 336, 337, 338, 339, 340, 342, 343, 344, 345, 346, 347, 348, 349, 351, 352, 354, 355, 356, 357, 358, 359, 360, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 374, 375, 377, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 393, 395, 396, 397, 398, 399, 400, 401, 403, 404, 405, 406, 407, 408, 409, 411, 412, 413, 414, 415, 417, 418, 419], "outer_test_idx": [7, 16, 20, 29, 51, 59, 60, 69, 70, 74, 78, 79, 82, 83, 84, 87, 91, 93, 97, 102, 110, 112, 119, 120, 121, 132, 137, 140, 142, 143, 152, 161, 162, 165, 166, 173, 174, 177, 181, 186, 187, 194, 205, 207, 209, 214, 215, 221, 223, 226, 235, 238, 240, 243, 246, 251, 254, 263, 264, 266, 273, 284, 289, 290, 293, 295, 302, 305, 314, 315, 331, 334, 341, 350, 353, 361, 373, 376, 378, 392, 394, 402, 410, 416]} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_0/test_metrics.json b/models/cv5_sample/moe_llm/seed42/outer_fold_0/test_metrics.json new file mode 100644 index 0000000..ea87e91 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_0/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 83, + "mse": 1.4858819540561683, + "rmse": 1.2189675771144073, + "mae": 0.5309677058494234, + "r2": 0.0906036692787614 + }, + "delivery": { + "n_samples": 58, + "mse": 0.4578579988449137, + "rmse": 0.6766520515338099, + "mae": 0.44371995239936074, + "r2": 0.41803485452290234 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7857142857142857, + "precision": 0.7322033898305085, + "recall": 0.7441197434069851, + "f1": 0.7375 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7380952380952381, + "precision": 0.7146208708708709, + "recall": 0.7547619047619047, + "f1": 0.7160330091364574 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9827586206896551, + "precision": 0.9912280701754386, + "recall": 0.75, + "f1": 0.8289085545722714 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.2124931399005794, + "js_divergence": 0.0436771104854127 + } +} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_1/best_params.json b/models/cv5_sample/moe_llm/seed42/outer_fold_1/best_params.json new file mode 100644 index 0000000..7395ce5 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_1/best_params.json @@ -0,0 +1,16 @@ +{ + "dropout": 0.19323529834582587, + "lr": 0.000980484886752915, + "weight_decay": 0.00014367509113739864, + "backbone_lr_ratio": 0.13093310334961422, + "moe_n_experts": 4, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_1/epoch_mean.json b/models/cv5_sample/moe_llm/seed42/outer_fold_1/epoch_mean.json new file mode 100644 index 0000000..735a6b9 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_1/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 11} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_1/history.json b/models/cv5_sample/moe_llm/seed42/outer_fold_1/history.json new file mode 100644 index 0000000..9378fdb --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_1/history.json @@ -0,0 +1,271 @@ +{ + "train": [ + { + "loss": 5.844873963175593, + "loss_size": 1.1697139462103714, + "loss_pdi": 0.6559584736824036, + "loss_ee": 0.9990178346633911, + "loss_delivery": 0.9326752996122515, + "loss_biodist": 0.9255792949650739, + "loss_toxic": 0.5104590783248076, + "loss_moe_lb": 2.0308946306641036, + "loss_aux_moe": 1.0037819767320477, + "loss_aux_llm": 1.1631469360074482 + }, + { + "loss": 4.553732060097359, + "loss_size": 1.1190044841653592, + "loss_pdi": 0.5833910236487517, + "loss_ee": 0.8829387386102934, + "loss_delivery": 0.7804326554005211, + "loss_biodist": 0.4103407849733894, + "loss_toxic": 0.30222512459432754, + "loss_moe_lb": 2.0332680811753145, + "loss_aux_moe": 0.8820628979922952, + "loss_aux_llm": 1.0768901111709106 + }, + { + "loss": 4.165987249967214, + "loss_size": 1.0680863821023219, + "loss_pdi": 0.5564941554456144, + "loss_ee": 0.8305287328926293, + "loss_delivery": 0.8199278615616463, + "loss_biodist": 0.30551903876098424, + "loss_toxic": 0.23546614855326511, + "loss_moe_lb": 2.1472520892684526, + "loss_aux_moe": 0.9134891435906694, + "loss_aux_llm": 1.0313057750463486 + }, + { + "loss": 3.887093511787621, + "loss_size": 1.103906466147384, + "loss_pdi": 0.5772485410844957, + "loss_ee": 0.8390914693072036, + "loss_delivery": 0.7768730807143289, + "loss_biodist": 0.21761337989890897, + "loss_toxic": 0.1712454239258895, + "loss_moe_lb": 2.1270299988823966, + "loss_aux_moe": 0.7972461388672929, + "loss_aux_llm": 1.0023344352438643 + }, + { + "loss": 3.5329479301297986, + "loss_size": 1.0075848493725061, + "loss_pdi": 0.5278141957682532, + "loss_ee": 0.7903922522390211, + "loss_delivery": 0.7493079117423779, + "loss_biodist": 0.216409088087243, + "loss_toxic": 0.14804497885366752, + "loss_moe_lb": 2.1215679355569788, + "loss_aux_moe": 0.7791867956921861, + "loss_aux_llm": 0.9144315647112357 + }, + { + "loss": 3.140749799238669, + "loss_size": 0.8968957497461422, + "loss_pdi": 0.4775858110672719, + "loss_ee": 0.7530861561362808, + "loss_delivery": 0.6636832944444708, + "loss_biodist": 0.1660661126512128, + "loss_toxic": 0.1779262529673508, + "loss_moe_lb": 2.1186544766297213, + "loss_aux_moe": 0.6614102075832922, + "loss_aux_llm": 0.9802552050835377 + }, + { + "loss": 2.982136368751526, + "loss_size": 0.8015535189896017, + "loss_pdi": 0.4799974121757456, + "loss_ee": 0.7338198085088987, + "loss_delivery": 0.6604716744575951, + "loss_biodist": 0.17151938232819777, + "loss_toxic": 0.1764316958488544, + "loss_moe_lb": 2.116116572070766 + }, + { + "loss": 2.6167806999103442, + "loss_size": 0.6552065660220545, + "loss_pdi": 0.4558954645653029, + "loss_ee": 0.6795905675437, + "loss_delivery": 0.5747650408664265, + "loss_biodist": 0.1658533130948608, + "loss_toxic": 0.12498937264663747, + "loss_moe_lb": 2.0987441733076766 + }, + { + "loss": 2.564945269275356, + "loss_size": 0.701622544812995, + "loss_pdi": 0.4624234595008799, + "loss_ee": 0.6582107753367037, + "loss_delivery": 0.58780188199032, + "loss_biodist": 0.16030761096123103, + "loss_toxic": 0.04463169629987631, + "loss_moe_lb": 2.111907923543775 + }, + { + "loss": 2.6338622086756938, + "loss_size": 0.7301617483230861, + "loss_pdi": 0.43557040232258876, + "loss_ee": 0.6678765420172665, + "loss_delivery": 0.6153083404375089, + "loss_biodist": 0.15970830488446597, + "loss_toxic": 0.07487160047025394, + "loss_moe_lb": 2.1209699076575204 + }, + { + "loss": 2.4515427640966467, + "loss_size": 0.6441191072399551, + "loss_pdi": 0.4289833254910804, + "loss_ee": 0.680172837263829, + "loss_delivery": 0.5573766311076848, + "loss_biodist": 0.14778587272441066, + "loss_toxic": 0.04419851102362815, + "loss_moe_lb": 2.0920171866545805 + } + ], + "val": [ + { + "loss": 2.8340189456939697, + "loss_size": 0.5454001704230904, + "loss_pdi": 0.6431448519229889, + "loss_ee": 0.9064770460128784, + "loss_delivery": 0.19316499456763267, + "loss_biodist": 0.4068158447742462, + "loss_toxic": 0.1233778491616249, + "loss_moe_lb": 2.032278299331665, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.7272727272727273, + "acc_toxic": 1.0 + }, + { + "loss": 2.3471620321273803, + "loss_size": 0.4725328356027603, + "loss_pdi": 0.6254292964935303, + "loss_ee": 0.7353704273700714, + "loss_delivery": 0.21038319170475006, + "loss_biodist": 0.2688927501440048, + "loss_toxic": 0.025847082026302814, + "loss_moe_lb": 2.2519073486328125, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.7575757575757576, + "acc_toxic": 1.0 + }, + { + "loss": 2.248750996589661, + "loss_size": 0.41793634593486784, + "loss_pdi": 0.6494660258293152, + "loss_ee": 0.6911145269870758, + "loss_delivery": 0.2551379919052124, + "loss_biodist": 0.23418427407741546, + "loss_toxic": 0.012842147471383214, + "loss_moe_lb": 2.597307586669922, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.7575757575757576, + "acc_toxic": 1.0 + }, + { + "loss": 2.2044697284698485, + "loss_size": 0.3363520873710513, + "loss_pdi": 0.6014128744602203, + "loss_ee": 0.7720682263374329, + "loss_delivery": 0.285314205288887, + "loss_biodist": 0.21102917790412903, + "loss_toxic": 0.024753990955650805, + "loss_moe_lb": 2.42631254196167, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.7272727272727273, + "acc_toxic": 1.0 + }, + { + "loss": 2.172011137008667, + "loss_size": 0.4511841475963593, + "loss_pdi": 0.6105897426605225, + "loss_ee": 0.7073207706212997, + "loss_delivery": 0.24147587269544601, + "loss_biodist": 0.18464208096265794, + "loss_toxic": 0.0065372010692954065, + "loss_moe_lb": 2.4683003425598145, + "acc_pdi": 0.6060606060606061, + "acc_ee": 0.7575757575757576, + "acc_toxic": 1.0 + }, + { + "loss": 2.072033905982971, + "loss_size": 0.4331493258476257, + "loss_pdi": 0.5731127798557282, + "loss_ee": 0.6349433034658432, + "loss_delivery": 0.2421866089105606, + "loss_biodist": 0.21265464127063752, + "loss_toxic": 0.0034104947932064532, + "loss_moe_lb": 2.45847373008728, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.7272727272727273, + "acc_toxic": 1.0 + }, + { + "loss": 2.0299159049987794, + "loss_size": 0.44897343069314954, + "loss_pdi": 0.5645411908626556, + "loss_ee": 0.6812305867671966, + "loss_delivery": 0.1939051305875182, + "loss_biodist": 0.17333385050296785, + "loss_toxic": 0.0041761530097574, + "loss_moe_lb": 2.4599356651306152, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.7272727272727273, + "acc_toxic": 1.0 + }, + { + "loss": 2.059905481338501, + "loss_size": 0.48319281786680224, + "loss_pdi": 0.5864004135131836, + "loss_ee": 0.6532717317342758, + "loss_delivery": 0.1758626704569906, + "loss_biodist": 0.19589750468730927, + "loss_toxic": 0.003806475317105651, + "loss_moe_lb": 2.442372369766235, + "acc_pdi": 0.7878787878787878, + "acc_ee": 0.7575757575757576, + "acc_toxic": 1.0 + }, + { + "loss": 2.0781530857086183, + "loss_size": 0.49511486291885376, + "loss_pdi": 0.5853226751089096, + "loss_ee": 0.6847641676664352, + "loss_delivery": 0.14788594823330642, + "loss_biodist": 0.20549408048391343, + "loss_toxic": 0.002917639911174774, + "loss_moe_lb": 2.4569051265716553, + "acc_pdi": 0.7878787878787878, + "acc_ee": 0.7575757575757576, + "acc_toxic": 1.0 + }, + { + "loss": 2.016889500617981, + "loss_size": 0.4801466763019562, + "loss_pdi": 0.5440871745347977, + "loss_ee": 0.6530755490064621, + "loss_delivery": 0.17224864036543294, + "loss_biodist": 0.20416025221347808, + "loss_toxic": 0.00327652373816818, + "loss_moe_lb": 2.4228747844696046, + "acc_pdi": 0.8484848484848485, + "acc_ee": 0.7575757575757576, + "acc_toxic": 1.0 + }, + { + "loss": 1.995576024055481, + "loss_size": 0.4439514249563217, + "loss_pdi": 0.5852887749671936, + "loss_ee": 0.6712759345769882, + "loss_delivery": 0.1613160498905927, + "loss_biodist": 0.17435047775506973, + "loss_toxic": 0.0036445653764531015, + "loss_moe_lb": 2.464684581756592, + "acc_pdi": 0.8181818181818182, + "acc_ee": 0.7575757575757576, + "acc_toxic": 1.0 + } + ] +} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_1/splits.json b/models/cv5_sample/moe_llm/seed42/outer_fold_1/splits.json new file mode 100644 index 0000000..1773fe5 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_1/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [0, 3, 5, 6, 7, 10, 11, 12, 13, 15, 16, 18, 19, 20, 21, 24, 26, 27, 29, 30, 31, 32, 33, 35, 36, 38, 39, 40, 42, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 65, 66, 67, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 97, 98, 99, 100, 101, 102, 103, 105, 106, 107, 108, 109, 110, 112, 114, 115, 118, 119, 120, 121, 123, 124, 125, 126, 128, 129, 130, 131, 132, 136, 137, 140, 141, 142, 143, 144, 145, 146, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 161, 162, 164, 165, 166, 167, 168, 169, 171, 172, 173, 174, 175, 176, 177, 178, 179, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 194, 195, 196, 197, 198, 199, 201, 202, 204, 205, 206, 207, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 225, 226, 228, 229, 231, 232, 233, 234, 235, 236, 237, 238, 240, 242, 243, 244, 246, 247, 248, 250, 251, 252, 254, 255, 256, 258, 259, 260, 261, 262, 263, 264, 266, 268, 269, 270, 271, 273, 274, 275, 276, 277, 278, 279, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 295, 296, 297, 300, 301, 302, 303, 304, 305, 306, 307, 308, 310, 311, 313, 314, 315, 316, 317, 318, 319, 321, 322, 323, 324, 325, 327, 328, 329, 330, 331, 333, 334, 335, 337, 338, 339, 340, 341, 342, 343, 344, 345, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 361, 362, 364, 365, 366, 367, 368, 369, 373, 374, 375, 376, 378, 379, 380, 381, 382, 384, 387, 388, 389, 391, 392, 393, 394, 396, 397, 398, 399, 402, 403, 404, 405, 406, 408, 409, 410, 411, 412, 413, 414, 416, 417, 419], "outer_test_idx": [1, 2, 4, 8, 9, 14, 17, 22, 23, 25, 28, 34, 37, 41, 43, 45, 63, 64, 68, 85, 96, 104, 111, 113, 116, 117, 122, 127, 133, 134, 135, 138, 139, 147, 160, 163, 170, 180, 191, 192, 193, 200, 203, 208, 224, 227, 230, 239, 241, 245, 249, 253, 257, 265, 267, 272, 280, 294, 298, 299, 309, 312, 320, 326, 332, 336, 346, 347, 360, 363, 370, 371, 372, 377, 383, 385, 386, 390, 395, 400, 401, 407, 415, 418]} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_1/test_metrics.json b/models/cv5_sample/moe_llm/seed42/outer_fold_1/test_metrics.json new file mode 100644 index 0000000..ba53be9 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_1/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.3521366695033162, + "rmse": 0.593411045990312, + "mae": 0.417994069290303, + "r2": -0.07530019609216088 + }, + "delivery": { + "n_samples": 61, + "mse": 0.8615718355352413, + "rmse": 0.9282089395902419, + "mae": 0.54733644703739, + "r2": 0.34785223029308054 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6309523809523809, + "precision": 0.5417771883289124, + "recall": 0.5476190476190477, + "f1": 0.5421135924037278 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5952380952380952, + "precision": 0.531971296677179, + "recall": 0.5323723895152467, + "f1": 0.5202020202020202 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9836065573770492, + "precision": 0.9915254237288136, + "recall": 0.8333333333333333, + "f1": 0.8957264957264958 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.12922332899139097, + "js_divergence": 0.030354793549857463 + } +} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_2/best_params.json b/models/cv5_sample/moe_llm/seed42/outer_fold_2/best_params.json new file mode 100644 index 0000000..7395ce5 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_2/best_params.json @@ -0,0 +1,16 @@ +{ + "dropout": 0.19323529834582587, + "lr": 0.000980484886752915, + "weight_decay": 0.00014367509113739864, + "backbone_lr_ratio": 0.13093310334961422, + "moe_n_experts": 4, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_2/epoch_mean.json b/models/cv5_sample/moe_llm/seed42/outer_fold_2/epoch_mean.json new file mode 100644 index 0000000..735a6b9 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_2/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 11} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_2/history.json b/models/cv5_sample/moe_llm/seed42/outer_fold_2/history.json new file mode 100644 index 0000000..85a9465 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_2/history.json @@ -0,0 +1,271 @@ +{ + "train": [ + { + "loss": 5.448386102109342, + "loss_size": 0.8643624792630608, + "loss_pdi": 0.6676707122776959, + "loss_ee": 1.0198738494434871, + "loss_delivery": 0.8485204742063541, + "loss_biodist": 0.9448029020348111, + "loss_toxic": 0.4876952078696844, + "loss_moe_lb": 2.093161576503032, + "loss_aux_moe": 0.8641154044181913, + "loss_aux_llm": 1.1632047895644162 + }, + { + "loss": 4.324384463799967, + "loss_size": 0.825573201517801, + "loss_pdi": 0.5958146203208614, + "loss_ee": 0.9189161110568691, + "loss_delivery": 0.7985159166761346, + "loss_biodist": 0.43780089069057154, + "loss_toxic": 0.2720214316772448, + "loss_moe_lb": 2.1437764683285274, + "loss_aux_moe": 0.8986711423542049, + "loss_aux_llm": 1.0598642354881442 + }, + { + "loss": 3.8106770064379716, + "loss_size": 0.7928319295515885, + "loss_pdi": 0.5804953180454873, + "loss_ee": 0.8457762880905254, + "loss_delivery": 0.7977923632070825, + "loss_biodist": 0.2861963832700575, + "loss_toxic": 0.14996869308320251, + "loss_moe_lb": 2.0927669937546187, + "loss_aux_moe": 0.9209618949004121, + "loss_aux_llm": 1.0866852402687073 + }, + { + "loss": 3.590217042613674, + "loss_size": 0.7697639918407878, + "loss_pdi": 0.6194508639541833, + "loss_ee": 0.8710581531395784, + "loss_delivery": 0.7570703561241562, + "loss_biodist": 0.2637654389884021, + "loss_toxic": 0.128640679239824, + "loss_moe_lb": 2.2199561015979663, + "loss_aux_moe": 0.7979112141639799, + "loss_aux_llm": 0.8974349112124056 + }, + { + "loss": 3.2631049865000956, + "loss_size": 0.7027561263860883, + "loss_pdi": 0.540183317822379, + "loss_ee": 0.8333719823811505, + "loss_delivery": 0.7394266468850342, + "loss_biodist": 0.20225958405314265, + "loss_toxic": 0.15996630871794312, + "loss_moe_lb": 2.2460104194847315, + "loss_aux_moe": 0.7544193904142122, + "loss_aux_llm": 0.8569866848757138 + }, + { + "loss": 2.9335989533243954, + "loss_size": 0.6793409892030664, + "loss_pdi": 0.5341143374507492, + "loss_ee": 0.8107952403055655, + "loss_delivery": 0.7013196065216452, + "loss_biodist": 0.17505165954699387, + "loss_toxic": 0.0537688604703903, + "loss_moe_lb": 2.211098677403218, + "loss_aux_moe": 0.6585364743463091, + "loss_aux_llm": 0.8589774281591982 + }, + { + "loss": 2.572078147450009, + "loss_size": 0.6022349424861573, + "loss_pdi": 0.5041297573495556, + "loss_ee": 0.7460332577292984, + "loss_delivery": 0.5461117913191383, + "loss_biodist": 0.1844338054793912, + "loss_toxic": 0.046531276912090205, + "loss_moe_lb": 2.26575459016336 + }, + { + "loss": 2.613092970203709, + "loss_size": 0.5582844796212943, + "loss_pdi": 0.5014023039792035, + "loss_ee": 0.7368455045932049, + "loss_delivery": 0.5232992003092894, + "loss_biodist": 0.1375063386921947, + "loss_toxic": 0.20026905486687413, + "loss_moe_lb": 2.2357002980000265 + }, + { + "loss": 2.414530187039762, + "loss_size": 0.5297113753653862, + "loss_pdi": 0.47427408356924317, + "loss_ee": 0.7108970058930887, + "loss_delivery": 0.5342327124766402, + "loss_biodist": 0.14476421435137052, + "loss_toxic": 0.07928779668401222, + "loss_moe_lb": 2.235232720503936 + }, + { + "loss": 2.374394999968039, + "loss_size": 0.5287634996143548, + "loss_pdi": 0.46810457754779505, + "loss_ee": 0.6844177930741697, + "loss_delivery": 0.527905018496755, + "loss_biodist": 0.18576635964013435, + "loss_toxic": 0.0401537462823546, + "loss_moe_lb": 2.2262662423623576 + }, + { + "loss": 2.186512160945583, + "loss_size": 0.4162472438167881, + "loss_pdi": 0.4552439733131512, + "loss_ee": 0.6912266974513596, + "loss_delivery": 0.4482238549437072, + "loss_biodist": 0.14196190117178736, + "loss_toxic": 0.08895144933941648, + "loss_moe_lb": 2.2699929765752844 + } + ], + "val": [ + { + "loss": 7.46802167892456, + "loss_size": 3.2265774309635162, + "loss_pdi": 0.7353087663650513, + "loss_ee": 1.0791295766830444, + "loss_delivery": 1.8187558829784394, + "loss_biodist": 0.45695706605911257, + "loss_toxic": 0.15167853236198425, + "loss_moe_lb": 2.1358907222747803, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.95 + }, + { + "loss": 7.090808343887329, + "loss_size": 3.410844311118126, + "loss_pdi": 0.7231593370437622, + "loss_ee": 1.0354070901870727, + "loss_delivery": 1.6264947444200515, + "loss_biodist": 0.2861800193786621, + "loss_toxic": 0.063536443002522, + "loss_moe_lb": 2.133509969711304, + "acc_pdi": 0.6363636363636364, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.95 + }, + { + "loss": 6.772978925704956, + "loss_size": 3.1503812193870546, + "loss_pdi": 0.7274282336235046, + "loss_ee": 1.085149884223938, + "loss_delivery": 1.600457674264908, + "loss_biodist": 0.2441837251186371, + "loss_toxic": 0.05247659794986248, + "loss_moe_lb": 2.238656997680664, + "acc_pdi": 0.6060606060606061, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.95 + }, + { + "loss": 6.508674764633179, + "loss_size": 3.038449627161026, + "loss_pdi": 0.7216925919055939, + "loss_ee": 1.0778231978416444, + "loss_delivery": 1.5543653905391692, + "loss_biodist": 0.14704219698905946, + "loss_toxic": 0.06883886940777302, + "loss_moe_lb": 2.328789567947388, + "acc_pdi": 0.6363636363636364, + "acc_ee": 0.5151515151515151, + "acc_toxic": 0.95 + }, + { + "loss": 6.362945938110352, + "loss_size": 3.0777094930410387, + "loss_pdi": 0.6802633225917816, + "loss_ee": 1.05179603099823, + "loss_delivery": 1.5254353404045105, + "loss_biodist": 0.10500667095184327, + "loss_toxic": 0.028991057851817457, + "loss_moe_lb": 2.4010870933532713, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.95 + }, + { + "loss": 5.968770718574524, + "loss_size": 2.767032378911972, + "loss_pdi": 0.6399851977825165, + "loss_ee": 1.0736777305603027, + "loss_delivery": 1.4773260951042175, + "loss_biodist": 0.11008006632328034, + "loss_toxic": 0.014215384027920663, + "loss_moe_lb": 2.367847776412964, + "acc_pdi": 0.6363636363636364, + "acc_ee": 0.5757575757575758, + "acc_toxic": 1.0 + }, + { + "loss": 6.202451419830322, + "loss_size": 3.0308651685714723, + "loss_pdi": 0.6326265096664428, + "loss_ee": 1.0713923573493958, + "loss_delivery": 1.4624701291322708, + "loss_biodist": 0.11829227358102798, + "loss_toxic": 0.0033330414851661773, + "loss_moe_lb": 2.3702832221984864, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.5151515151515151, + "acc_toxic": 1.0 + }, + { + "loss": 6.452511215209961, + "loss_size": 3.152136731147766, + "loss_pdi": 0.7097072839736939, + "loss_ee": 1.0254977822303772, + "loss_delivery": 1.543718020617962, + "loss_biodist": 0.13677483052015305, + "loss_toxic": 0.0033543511119205503, + "loss_moe_lb": 2.3397052764892576, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.5454545454545454, + "acc_toxic": 1.0 + }, + { + "loss": 6.112395906448365, + "loss_size": 2.9286181777715683, + "loss_pdi": 0.6871815204620362, + "loss_ee": 1.0443474650382996, + "loss_delivery": 1.444450181722641, + "loss_biodist": 0.12184233590960503, + "loss_toxic": 0.004058638200513087, + "loss_moe_lb": 2.3976557731628416, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.6060606060606061, + "acc_toxic": 1.0 + }, + { + "loss": 6.221766138076783, + "loss_size": 3.082172819972038, + "loss_pdi": 0.670418244600296, + "loss_ee": 1.0383009672164918, + "loss_delivery": 1.4141629114747047, + "loss_biodist": 0.1265248589217663, + "loss_toxic": 0.005223917227704078, + "loss_moe_lb": 2.367207431793213, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.5757575757575758, + "acc_toxic": 1.0 + }, + { + "loss": 6.086094331741333, + "loss_size": 3.0313263446092606, + "loss_pdi": 0.6579509675502777, + "loss_ee": 1.0429080128669739, + "loss_delivery": 1.34516292065382, + "loss_biodist": 0.1171148531138897, + "loss_toxic": 0.005285785451997071, + "loss_moe_lb": 2.3800329685211183, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.5757575757575758, + "acc_toxic": 1.0 + } + ] +} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_2/splits.json b/models/cv5_sample/moe_llm/seed42/outer_fold_2/splits.json new file mode 100644 index 0000000..14df306 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_2/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 36, 37, 38, 39, 40, 41, 43, 44, 45, 47, 48, 49, 50, 51, 55, 56, 57, 58, 59, 60, 61, 63, 64, 65, 68, 69, 70, 72, 74, 76, 78, 79, 80, 82, 83, 84, 85, 86, 87, 88, 91, 93, 94, 95, 96, 97, 100, 101, 102, 103, 104, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 142, 143, 145, 147, 148, 149, 150, 151, 152, 153, 156, 157, 158, 159, 160, 161, 162, 163, 165, 166, 167, 168, 170, 171, 172, 173, 174, 175, 177, 179, 180, 181, 183, 184, 185, 186, 187, 188, 191, 192, 193, 194, 196, 197, 199, 200, 202, 203, 204, 205, 207, 208, 209, 210, 211, 212, 213, 214, 215, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 232, 233, 235, 236, 237, 238, 239, 240, 241, 242, 243, 245, 246, 249, 250, 251, 252, 253, 254, 255, 256, 257, 259, 260, 261, 263, 264, 265, 266, 267, 268, 269, 271, 272, 273, 274, 275, 276, 280, 281, 282, 283, 284, 285, 287, 289, 290, 293, 294, 295, 296, 297, 298, 299, 300, 302, 303, 305, 307, 308, 309, 310, 311, 312, 313, 314, 315, 317, 318, 320, 321, 322, 323, 324, 326, 327, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 358, 360, 361, 363, 364, 365, 366, 368, 369, 370, 371, 372, 373, 376, 377, 378, 379, 380, 381, 382, 383, 385, 386, 387, 388, 389, 390, 392, 394, 395, 398, 399, 400, 401, 402, 404, 405, 407, 408, 409, 410, 412, 413, 414, 415, 416, 417, 418], "outer_test_idx": [10, 12, 15, 31, 35, 42, 46, 52, 53, 54, 62, 66, 67, 71, 73, 75, 77, 81, 89, 90, 92, 98, 99, 105, 123, 124, 141, 144, 146, 154, 155, 164, 169, 176, 178, 182, 189, 190, 195, 198, 201, 206, 216, 217, 218, 219, 231, 234, 244, 247, 248, 258, 262, 270, 277, 278, 279, 286, 288, 291, 292, 301, 304, 306, 316, 319, 325, 328, 343, 357, 359, 362, 367, 374, 375, 384, 391, 393, 396, 397, 403, 406, 411, 419]} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_2/test_metrics.json b/models/cv5_sample/moe_llm/seed42/outer_fold_2/test_metrics.json new file mode 100644 index 0000000..9df1155 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_2/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.4170957202870868, + "rmse": 0.6458294823613171, + "mae": 0.4985973424876907, + "r2": 0.1762434046684933 + }, + "delivery": { + "n_samples": 60, + "mse": 0.5854544655957109, + "rmse": 0.7651499628149445, + "mae": 0.49963968732239056, + "r2": 0.22065863304023947 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7380952380952381, + "precision": 0.6851851851851851, + "recall": 0.7199413489736071, + "f1": 0.693633952254642 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6666666666666666, + "precision": 0.613035584604212, + "recall": 0.6666666666666666, + "f1": 0.627436739505705 + }, + "toxic": { + "n_samples": 61, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.17218262970209255, + "js_divergence": 0.039882423974733884 + } +} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_3/best_params.json b/models/cv5_sample/moe_llm/seed42/outer_fold_3/best_params.json new file mode 100644 index 0000000..7395ce5 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_3/best_params.json @@ -0,0 +1,16 @@ +{ + "dropout": 0.19323529834582587, + "lr": 0.000980484886752915, + "weight_decay": 0.00014367509113739864, + "backbone_lr_ratio": 0.13093310334961422, + "moe_n_experts": 4, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_3/epoch_mean.json b/models/cv5_sample/moe_llm/seed42/outer_fold_3/epoch_mean.json new file mode 100644 index 0000000..735a6b9 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_3/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 11} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_3/history.json b/models/cv5_sample/moe_llm/seed42/outer_fold_3/history.json new file mode 100644 index 0000000..ba8a128 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_3/history.json @@ -0,0 +1,271 @@ +{ + "train": [ + { + "loss": 5.186875691285005, + "loss_size": 0.5132114932545133, + "loss_pdi": 0.6726247926016111, + "loss_ee": 1.0378613858609587, + "loss_delivery": 0.9076983817928547, + "loss_biodist": 0.9157768181852393, + "loss_toxic": 0.5092847890950538, + "loss_moe_lb": 2.0349878427144645, + "loss_aux_moe": 1.010407390223967, + "loss_aux_llm": 1.083440064981177 + }, + { + "loss": 3.907044636236655, + "loss_size": 0.45616640432460887, + "loss_pdi": 0.6273065104677871, + "loss_ee": 0.9086244025745908, + "loss_delivery": 0.8223057431546418, + "loss_biodist": 0.37386319846720306, + "loss_toxic": 0.2553665132538692, + "loss_moe_lb": 2.043943031414135, + "loss_aux_moe": 0.9434679752668819, + "loss_aux_llm": 0.9746431202904599 + }, + { + "loss": 3.3520812343906714, + "loss_size": 0.43402249788915787, + "loss_pdi": 0.5923634042610993, + "loss_ee": 0.8609370621475013, + "loss_delivery": 0.7527698189825625, + "loss_biodist": 0.23749435713162292, + "loss_toxic": 0.14371532126254327, + "loss_moe_lb": 2.1175068971273063, + "loss_aux_moe": 0.9599171609089181, + "loss_aux_llm": 0.9244846647252908 + }, + { + "loss": 3.13085011855976, + "loss_size": 0.4183622161681588, + "loss_pdi": 0.5564500198170945, + "loss_ee": 0.8595020851573428, + "loss_delivery": 0.7666488471063407, + "loss_biodist": 0.2030809574030541, + "loss_toxic": 0.13739074190852005, + "loss_moe_lb": 2.0961369018296936, + "loss_aux_moe": 0.8233248805375518, + "loss_aux_llm": 0.9434398358335366 + }, + { + "loss": 2.7858061146091773, + "loss_size": 0.39161805264853145, + "loss_pdi": 0.566842187900801, + "loss_ee": 0.7711971643808726, + "loss_delivery": 0.6954803979527708, + "loss_biodist": 0.18086066509823542, + "loss_toxic": 0.09432240759928685, + "loss_moe_lb": 2.094309987248601, + "loss_aux_moe": 0.7374768013486991, + "loss_aux_llm": 0.9944169878758289 + }, + { + "loss": 2.470173813201286, + "loss_size": 0.3628059673953701, + "loss_pdi": 0.5112751488750046, + "loss_ee": 0.7295194646796664, + "loss_delivery": 0.6388976760208607, + "loss_biodist": 0.16272256372345462, + "loss_toxic": 0.0796801333057669, + "loss_moe_lb": 2.0946719453141496, + "loss_aux_moe": 0.6715661742397256, + "loss_aux_llm": 0.8760432788246387 + }, + { + "loss": 2.361145433541891, + "loss_size": 0.3760294072531365, + "loss_pdi": 0.5063537709616326, + "loss_ee": 0.7174645541487513, + "loss_delivery": 0.5734429028187249, + "loss_biodist": 0.15295277354685036, + "loss_toxic": 0.09341773955698905, + "loss_moe_lb": 2.107072810868959 + }, + { + "loss": 2.285447710269206, + "loss_size": 0.33821960578899124, + "loss_pdi": 0.5100937501804249, + "loss_ee": 0.7393307581141189, + "loss_delivery": 0.5426343622038493, + "loss_biodist": 0.12398734537733568, + "loss_toxic": 0.09918186348875531, + "loss_moe_lb": 2.0946988995010787 + }, + { + "loss": 2.1923618091119303, + "loss_size": 0.3291457059818345, + "loss_pdi": 0.48687964556990443, + "loss_ee": 0.7320111675842388, + "loss_delivery": 0.5369202897355363, + "loss_biodist": 0.10976744191469373, + "loss_toxic": 0.0715399857372914, + "loss_moe_lb": 2.107653498649597 + }, + { + "loss": 2.1388532954293327, + "loss_size": 0.289924700970988, + "loss_pdi": 0.4686911090000256, + "loss_ee": 0.6864962835569639, + "loss_delivery": 0.5863513408883197, + "loss_biodist": 0.12377736652018251, + "loss_toxic": 0.05824326928956686, + "loss_moe_lb": 2.0926922881925427 + }, + { + "loss": 2.0976792155085384, + "loss_size": 0.31547431859212954, + "loss_pdi": 0.47137918746149216, + "loss_ee": 0.6413027813305726, + "loss_delivery": 0.57016261661979, + "loss_biodist": 0.12815634000140266, + "loss_toxic": 0.04252448489077742, + "loss_moe_lb": 2.0866256404567407 + } + ], + "val": [ + { + "loss": 5.893757343292236, + "loss_size": 2.7006002753973006, + "loss_pdi": 0.6564314365386963, + "loss_ee": 0.8844664573669434, + "loss_delivery": 0.9374306380748749, + "loss_biodist": 0.38643685579299925, + "loss_toxic": 0.312559700012207, + "loss_moe_lb": 2.0757519721984865, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.696969696969697, + "acc_toxic": 0.782608695652174 + }, + { + "loss": 5.338843441009521, + "loss_size": 2.6443060830235483, + "loss_pdi": 0.6313182950019837, + "loss_ee": 0.7473037540912628, + "loss_delivery": 0.9139358669519424, + "loss_biodist": 0.16414627730846404, + "loss_toxic": 0.21320655420422555, + "loss_moe_lb": 2.070327711105347, + "acc_pdi": 0.7272727272727273, + "acc_ee": 0.6666666666666666, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 5.012914252281189, + "loss_size": 2.5547497063875197, + "loss_pdi": 0.5505357623100281, + "loss_ee": 0.6987002611160278, + "loss_delivery": 0.823940533399582, + "loss_biodist": 0.15457155257463456, + "loss_toxic": 0.19016381772235036, + "loss_moe_lb": 2.116373634338379, + "acc_pdi": 0.7878787878787878, + "acc_ee": 0.6363636363636364, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 5.13802444934845, + "loss_size": 2.5034602016210554, + "loss_pdi": 0.554446917772293, + "loss_ee": 0.737292206287384, + "loss_delivery": 0.9933651030063629, + "loss_biodist": 0.12027869075536728, + "loss_toxic": 0.2056742062792182, + "loss_moe_lb": 2.114448404312134, + "acc_pdi": 0.7878787878787878, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 5.1256492137908936, + "loss_size": 2.59128378033638, + "loss_pdi": 0.5555723905563354, + "loss_ee": 0.8101335763931274, + "loss_delivery": 0.8393854141235352, + "loss_biodist": 0.1212102085351944, + "loss_toxic": 0.18476905608549715, + "loss_moe_lb": 2.1382421493530273, + "acc_pdi": 0.7878787878787878, + "acc_ee": 0.6666666666666666, + "acc_toxic": 0.8695652173913043 + }, + { + "loss": 4.646122789382934, + "loss_size": 2.2567166566848753, + "loss_pdi": 0.5647770702838898, + "loss_ee": 0.8054781496524811, + "loss_delivery": 0.8111683785915375, + "loss_biodist": 0.11808478087186813, + "loss_toxic": 0.08715959303081036, + "loss_moe_lb": 2.135502338409424, + "acc_pdi": 0.7878787878787878, + "acc_ee": 0.6363636363636364, + "acc_toxic": 0.9130434782608695 + }, + { + "loss": 4.743265771865845, + "loss_size": 2.4424239844083786, + "loss_pdi": 0.5591305613517761, + "loss_ee": 0.869558048248291, + "loss_delivery": 0.6986709222197532, + "loss_biodist": 0.11900895535945892, + "loss_toxic": 0.04972788635641336, + "loss_moe_lb": 2.146485376358032, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.5757575757575758, + "acc_toxic": 1.0 + }, + { + "loss": 4.451839828491211, + "loss_size": 2.216589081287384, + "loss_pdi": 0.4908704340457916, + "loss_ee": 0.8932434380054474, + "loss_delivery": 0.6621725678443908, + "loss_biodist": 0.11659160852432252, + "loss_toxic": 0.07040527649223804, + "loss_moe_lb": 2.1616697788238524, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.9565217391304348 + }, + { + "loss": 4.532372415065765, + "loss_size": 2.252890150249004, + "loss_pdi": 0.5175323605537414, + "loss_ee": 0.9039882540702819, + "loss_delivery": 0.6627729907631874, + "loss_biodist": 0.12313011735677719, + "loss_toxic": 0.0710445934906602, + "loss_moe_lb": 2.1507219314575194, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.5454545454545454, + "acc_toxic": 1.0 + }, + { + "loss": 4.523290491104126, + "loss_size": 2.22698954641819, + "loss_pdi": 0.5167619049549103, + "loss_ee": 0.8895878970623017, + "loss_delivery": 0.6798219382762909, + "loss_biodist": 0.12791793793439865, + "loss_toxic": 0.07941748769953846, + "loss_moe_lb": 2.1426663398742676, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.9130434782608695 + }, + { + "loss": 4.4532647132873535, + "loss_size": 2.1416719794273376, + "loss_pdi": 0.522536426782608, + "loss_ee": 0.9024902045726776, + "loss_delivery": 0.681278795003891, + "loss_biodist": 0.11594324558973312, + "loss_toxic": 0.091428601462394, + "loss_moe_lb": 2.1225868701934814, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.9130434782608695 + } + ] +} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_3/splits.json b/models/cv5_sample/moe_llm/seed42/outer_fold_3/splits.json new file mode 100644 index 0000000..7a2b13e --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_3/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [0, 1, 2, 4, 5, 7, 8, 9, 10, 11, 12, 14, 15, 16, 17, 20, 21, 22, 23, 24, 25, 27, 28, 29, 30, 31, 32, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 45, 46, 47, 51, 52, 53, 54, 55, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 73, 74, 75, 76, 77, 78, 79, 81, 82, 83, 84, 85, 86, 87, 89, 90, 91, 92, 93, 95, 96, 97, 98, 99, 100, 102, 104, 105, 106, 109, 110, 111, 112, 113, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 127, 129, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 146, 147, 148, 150, 151, 152, 153, 154, 155, 156, 158, 159, 160, 161, 162, 163, 164, 165, 166, 168, 169, 170, 172, 173, 174, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 189, 190, 191, 192, 193, 194, 195, 198, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 212, 213, 214, 215, 216, 217, 218, 219, 221, 223, 224, 226, 227, 228, 230, 231, 233, 234, 235, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 251, 252, 253, 254, 255, 256, 257, 258, 259, 262, 263, 264, 265, 266, 267, 268, 269, 270, 272, 273, 274, 275, 276, 277, 278, 279, 280, 282, 284, 286, 288, 289, 290, 291, 292, 293, 294, 295, 297, 298, 299, 301, 302, 304, 305, 306, 307, 308, 309, 310, 312, 314, 315, 316, 317, 319, 320, 321, 325, 326, 327, 328, 331, 332, 334, 336, 340, 341, 343, 344, 346, 347, 348, 350, 352, 353, 357, 359, 360, 361, 362, 363, 364, 366, 367, 368, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 382, 383, 384, 385, 386, 387, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 400, 401, 402, 403, 404, 406, 407, 410, 411, 412, 414, 415, 416, 418, 419], "outer_test_idx": [3, 6, 13, 18, 19, 26, 33, 44, 48, 49, 50, 56, 57, 72, 80, 88, 94, 101, 103, 107, 108, 114, 126, 128, 130, 145, 149, 157, 167, 171, 175, 188, 196, 197, 199, 211, 220, 222, 225, 229, 232, 236, 250, 260, 261, 271, 281, 283, 285, 287, 296, 300, 303, 311, 313, 318, 322, 323, 324, 329, 330, 333, 335, 337, 338, 339, 342, 345, 349, 351, 354, 355, 356, 358, 365, 369, 381, 388, 399, 405, 408, 409, 413, 417]} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_3/test_metrics.json b/models/cv5_sample/moe_llm/seed42/outer_fold_3/test_metrics.json new file mode 100644 index 0000000..9be6476 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_3/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 83, + "mse": 1.773779594197195, + "rmse": 1.331833170557482, + "mae": 0.5606547438887408, + "r2": -0.003263258057283336 + }, + "delivery": { + "n_samples": 59, + "mse": 0.6480387215025862, + "rmse": 0.8050085226272988, + "mae": 0.5227914613403254, + "r2": 0.37002936863191194 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.75, + "precision": 0.6789023521026372, + "recall": 0.684017595307918, + "f1": 0.6813008130081302 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6428571428571429, + "precision": 0.5700750584471516, + "recall": 0.5891402714932127, + "f1": 0.5736800630417652 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9833333333333333, + "precision": 0.875, + "recall": 0.9912280701754386, + "f1": 0.9241466498103665 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.15399347135454697, + "js_divergence": 0.036819733414228055 + } +} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_4/best_params.json b/models/cv5_sample/moe_llm/seed42/outer_fold_4/best_params.json new file mode 100644 index 0000000..7395ce5 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_4/best_params.json @@ -0,0 +1,16 @@ +{ + "dropout": 0.19323529834582587, + "lr": 0.000980484886752915, + "weight_decay": 0.00014367509113739864, + "backbone_lr_ratio": 0.13093310334961422, + "moe_n_experts": 4, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" +} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_4/epoch_mean.json b/models/cv5_sample/moe_llm/seed42/outer_fold_4/epoch_mean.json new file mode 100644 index 0000000..735a6b9 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_4/epoch_mean.json @@ -0,0 +1 @@ +{"epoch_mean": 11} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_4/history.json b/models/cv5_sample/moe_llm/seed42/outer_fold_4/history.json new file mode 100644 index 0000000..69bdc8a --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_4/history.json @@ -0,0 +1,271 @@ +{ + "train": [ + { + "loss": 5.859205187977971, + "loss_size": 1.117435747103111, + "loss_pdi": 0.67151229768186, + "loss_ee": 1.0485850894773328, + "loss_delivery": 0.9128926674256453, + "loss_biodist": 0.9507973387434676, + "loss_toxic": 0.5287212576415088, + "loss_moe_lb": 2.1035370407877743, + "loss_aux_moe": 0.964274688951067, + "loss_aux_llm": 1.1332960209330998 + }, + { + "loss": 4.493758427130209, + "loss_size": 1.0583569308793224, + "loss_pdi": 0.6126068854654158, + "loss_ee": 0.9225417810517389, + "loss_delivery": 0.8066660902790121, + "loss_biodist": 0.4459190773400101, + "loss_toxic": 0.22362688823124846, + "loss_moe_lb": 2.0593858892853194, + "loss_aux_moe": 0.8353829867131001, + "loss_aux_llm": 0.9469703129819922 + }, + { + "loss": 3.938486898267591, + "loss_size": 0.9910998611232719, + "loss_pdi": 0.5878341842342067, + "loss_ee": 0.8390434464892825, + "loss_delivery": 0.7174631838460226, + "loss_biodist": 0.2919453759048436, + "loss_toxic": 0.1582938922267105, + "loss_moe_lb": 2.0062548792040027, + "loss_aux_moe": 0.9655985453644315, + "loss_aux_llm": 1.0280679303246576 + }, + { + "loss": 3.858765460349418, + "loss_size": 0.9830634577854259, + "loss_pdi": 0.5594958969064661, + "loss_ee": 0.8622410829002792, + "loss_delivery": 0.814423064525063, + "loss_biodist": 0.2729547468794359, + "loss_toxic": 0.16985372757237102, + "loss_moe_lb": 2.0284129091211267, + "loss_aux_moe": 0.7865037793243254, + "loss_aux_llm": 1.015865511588148 + }, + { + "loss": 3.2949182278401143, + "loss_size": 0.9350303498071593, + "loss_pdi": 0.5480591703105617, + "loss_ee": 0.7708510216828939, + "loss_delivery": 0.6737078446391467, + "loss_biodist": 0.18565125703006177, + "loss_toxic": 0.0994621277031665, + "loss_moe_lb": 2.0545133126748576, + "loss_aux_moe": 0.682390890713479, + "loss_aux_llm": 0.9088856340260119 + }, + { + "loss": 3.1421183669889294, + "loss_size": 0.9002627499804303, + "loss_pdi": 0.540199404632723, + "loss_ee": 0.7999422775732504, + "loss_delivery": 0.6205245248369269, + "loss_biodist": 0.20558074076433439, + "loss_toxic": 0.089201840867817, + "loss_moe_lb": 2.057482020275013, + "loss_aux_moe": 0.68815186116341, + "loss_aux_llm": 0.8820263811865369 + }, + { + "loss": 2.9004717743074573, + "loss_size": 0.7811062932014465, + "loss_pdi": 0.5034840235839019, + "loss_ee": 0.7158847797561336, + "loss_delivery": 0.6796143882580705, + "loss_biodist": 0.17312858465152817, + "loss_toxic": 0.09950823990024023, + "loss_moe_lb": 2.035009638683216 + }, + { + "loss": 2.9226431202244116, + "loss_size": 0.8814810771193053, + "loss_pdi": 0.5112376519151636, + "loss_ee": 0.7327054779271822, + "loss_delivery": 0.591531743874421, + "loss_biodist": 0.16855915958011472, + "loss_toxic": 0.0948821196735308, + "loss_moe_lb": 2.0614523597665735 + }, + { + "loss": 2.655469871856071, + "loss_size": 0.760228231347896, + "loss_pdi": 0.48637631092522593, + "loss_ee": 0.6979158569026638, + "loss_delivery": 0.5602184928752281, + "loss_biodist": 0.14575726085820714, + "loss_toxic": 0.06323230568306974, + "loss_moe_lb": 2.050211484367783 + }, + { + "loss": 2.509097969209826, + "loss_size": 0.7080223632825388, + "loss_pdi": 0.4473253053587836, + "loss_ee": 0.6626571416854858, + "loss_delivery": 0.5532580835593713, + "loss_biodist": 0.1437241429009953, + "loss_toxic": 0.04900663166709647, + "loss_moe_lb": 2.079811640687891 + }, + { + "loss": 2.484670532716287, + "loss_size": 0.7306602957683641, + "loss_pdi": 0.47313058899866567, + "loss_ee": 0.660492988454329, + "loss_delivery": 0.5034271306484133, + "loss_biodist": 0.13187124368709488, + "loss_toxic": 0.04232822387869394, + "loss_moe_lb": 2.062951352145221 + } + ], + "val": [ + { + "loss": 3.755842113494873, + "loss_size": 0.4265119343996048, + "loss_pdi": 0.5915934443473816, + "loss_ee": 0.8747140049934388, + "loss_delivery": 0.8283049013465643, + "loss_biodist": 0.6451014578342438, + "loss_toxic": 0.39165826290845873, + "loss_moe_lb": 2.1249768257141115, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.932078742980957, + "loss_size": 0.3444555729627609, + "loss_pdi": 0.5126577079296112, + "loss_ee": 0.7018583565950394, + "loss_delivery": 0.8085120553150773, + "loss_biodist": 0.24770227670669556, + "loss_toxic": 0.309023617208004, + "loss_moe_lb": 2.076921796798706, + "acc_pdi": 0.5151515151515151, + "acc_ee": 0.6060606060606061, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.628650116920471, + "loss_size": 0.31893138736486437, + "loss_pdi": 0.5060839474201202, + "loss_ee": 0.6603254437446594, + "loss_delivery": 0.6911659525707364, + "loss_biodist": 0.16578628718852997, + "loss_toxic": 0.28838122070301325, + "loss_moe_lb": 2.0901700019836427, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.6363636363636364, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.635863518714905, + "loss_size": 0.21646426171064376, + "loss_pdi": 0.4807057544589043, + "loss_ee": 0.7196594156324864, + "loss_delivery": 0.7288934513926506, + "loss_biodist": 0.12086161077022553, + "loss_toxic": 0.37542195692658425, + "loss_moe_lb": 2.13119113445282, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 2.6391834259033202, + "loss_size": 0.25473385751247407, + "loss_pdi": 0.515596741437912, + "loss_ee": 0.7078545540571213, + "loss_delivery": 0.7006262980401516, + "loss_biodist": 0.13355249539017677, + "loss_toxic": 0.3424140252172947, + "loss_moe_lb": 2.1053977251052856, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.8888888888888888 + }, + { + "loss": 2.775922429561615, + "loss_size": 0.26124624609947206, + "loss_pdi": 0.46465142369270324, + "loss_ee": 0.7462939266115427, + "loss_delivery": 0.699479284894187, + "loss_biodist": 0.10829939097166061, + "loss_toxic": 0.5012864658609033, + "loss_moe_lb": 2.1504496335983276, + "acc_pdi": 0.7575757575757576, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.565546917915344, + "loss_size": 0.23962730169296265, + "loss_pdi": 0.46498828530311587, + "loss_ee": 0.8164169616997242, + "loss_delivery": 0.734196837246418, + "loss_biodist": 0.09018271192908286, + "loss_toxic": 0.2623913433402777, + "loss_moe_lb": 2.1556874752044677, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.461030912399292, + "loss_size": 0.20080993920564652, + "loss_pdi": 0.4726409152150154, + "loss_ee": 0.8052330955862999, + "loss_delivery": 0.7270681807538495, + "loss_biodist": 0.08629608154296875, + "loss_toxic": 0.21949858171865344, + "loss_moe_lb": 2.152100443840027, + "acc_pdi": 0.696969696969697, + "acc_ee": 0.5454545454545454, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.446087622642517, + "loss_size": 0.2072613552212715, + "loss_pdi": 0.4464923307299614, + "loss_ee": 0.8108729934319854, + "loss_delivery": 0.7515319633297622, + "loss_biodist": 0.08003018759191036, + "loss_toxic": 0.20465249773114919, + "loss_moe_lb": 2.1997459888458253, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.5454545454545454, + "acc_toxic": 1.0 + }, + { + "loss": 2.4781503915786742, + "loss_size": 0.19303087145090103, + "loss_pdi": 0.45752228796482086, + "loss_ee": 0.7772767193615436, + "loss_delivery": 0.777904954974656, + "loss_biodist": 0.09190915115177631, + "loss_toxic": 0.23136587778571993, + "loss_moe_lb": 2.1780391693115235, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.9444444444444444 + }, + { + "loss": 2.4587207794189454, + "loss_size": 0.1924375042319298, + "loss_pdi": 0.4666991338133812, + "loss_ee": 0.770481814071536, + "loss_delivery": 0.7665328018600122, + "loss_biodist": 0.09240943193435669, + "loss_toxic": 0.22187743864487858, + "loss_moe_lb": 2.1950268745422363, + "acc_pdi": 0.6666666666666666, + "acc_ee": 0.5757575757575758, + "acc_toxic": 0.9444444444444444 + } + ] +} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_4/splits.json b/models/cv5_sample/moe_llm/seed42/outer_fold_4/splits.json new file mode 100644 index 0000000..8dd0299 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_4/splits.json @@ -0,0 +1 @@ +{"outer_train_idx": [1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23, 25, 26, 28, 29, 31, 33, 34, 35, 37, 41, 42, 43, 44, 45, 46, 48, 49, 50, 51, 52, 53, 54, 56, 57, 59, 60, 62, 63, 64, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 77, 78, 79, 80, 81, 82, 83, 84, 85, 87, 88, 89, 90, 91, 92, 93, 94, 96, 97, 98, 99, 101, 102, 103, 104, 105, 107, 108, 110, 111, 112, 113, 114, 116, 117, 119, 120, 121, 122, 123, 124, 126, 127, 128, 130, 132, 133, 134, 135, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 149, 152, 154, 155, 157, 160, 161, 162, 163, 164, 165, 166, 167, 169, 170, 171, 173, 174, 175, 176, 177, 178, 180, 181, 182, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 203, 205, 206, 207, 208, 209, 211, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 229, 230, 231, 232, 234, 235, 236, 238, 239, 240, 241, 243, 244, 245, 246, 247, 248, 249, 250, 251, 253, 254, 257, 258, 260, 261, 262, 263, 264, 265, 266, 267, 270, 271, 272, 273, 277, 278, 279, 280, 281, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 298, 299, 300, 301, 302, 303, 304, 305, 306, 309, 311, 312, 313, 314, 315, 316, 318, 319, 320, 322, 323, 324, 325, 326, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 341, 342, 343, 345, 346, 347, 349, 350, 351, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 365, 367, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 381, 383, 384, 385, 386, 388, 390, 391, 392, 393, 394, 395, 396, 397, 399, 400, 401, 402, 403, 405, 406, 407, 408, 409, 410, 411, 413, 415, 416, 417, 418, 419], "outer_test_idx": [0, 5, 11, 21, 24, 27, 30, 32, 36, 38, 39, 40, 47, 55, 58, 61, 65, 76, 86, 95, 100, 106, 109, 115, 118, 125, 129, 131, 136, 148, 150, 151, 153, 156, 158, 159, 168, 172, 179, 183, 184, 185, 202, 204, 210, 212, 213, 228, 233, 237, 242, 252, 255, 256, 259, 268, 269, 274, 275, 276, 282, 297, 307, 308, 310, 317, 321, 327, 340, 344, 348, 352, 364, 366, 368, 379, 380, 382, 387, 389, 398, 404, 412, 414]} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/outer_fold_4/test_metrics.json b/models/cv5_sample/moe_llm/seed42/outer_fold_4/test_metrics.json new file mode 100644 index 0000000..7f6d44e --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/outer_fold_4/test_metrics.json @@ -0,0 +1,42 @@ +{ + "size": { + "n_samples": 84, + "mse": 0.6805742520206829, + "rmse": 0.8249692430755724, + "mae": 0.4822031688111435, + "r2": 0.08930711264607172 + }, + "delivery": { + "n_samples": 58, + "mse": 0.7669285932160084, + "rmse": 0.8757445935979328, + "mae": 0.567918715805843, + "r2": 0.2583927108955246 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7619047619047619, + "precision": 0.7002652519893899, + "recall": 0.7214076246334311, + "f1": 0.7083333333333334 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7023809523809523, + "precision": 0.6327380952380953, + "recall": 0.6816239316239318, + "f1": 0.6431159420289855 + }, + "toxic": { + "n_samples": 59, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.16534984510664474, + "js_divergence": 0.041883856181021084 + } +} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/strata_info.json b/models/cv5_sample/moe_llm/seed42/strata_info.json new file mode 100644 index 0000000..ef3be58 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/strata_info.json @@ -0,0 +1,60 @@ +{ + "original_strata_counts": { + "T0|P0|E0": "5", + "T0|P0|E1": "58", + "T0|P0|E2": "169", + "T0|P1|E0": "1", + "T0|P1|E1": "17", + "T0|P1|E2": "32", + "T0|P2|E2": "3", + "T1|P0|E2": "9", + "T1|P1|E2": "5", + "TNA|P0|E0": "28", + "TNA|P0|E1": "21", + "TNA|P0|E2": "20", + "TNA|P1|E0": "29", + "TNA|P1|E1": "7", + "TNA|P1|E2": "14", + "TNA|P2|E2": "1", + "TNA|P3|E0": "1" + }, + "rare_strata": [ + "T0|P1|E0", + "T0|P2|E2", + "TNA|P2|E2", + "TNA|P3|E0" + ], + "final_strata": [ + "RARE", + "T0|P0|E0", + "T0|P0|E1", + "T0|P0|E2", + "T0|P1|E1", + "T0|P1|E2", + "T1|P0|E2", + "T1|P1|E2", + "TNA|P0|E0", + "TNA|P0|E1", + "TNA|P0|E2", + "TNA|P1|E0", + "TNA|P1|E1", + "TNA|P1|E2" + ], + "final_strata_counts": { + "RARE": "6", + "T0|P0|E0": "5", + "T0|P0|E1": "58", + "T0|P0|E2": "169", + "T0|P1|E1": "17", + "T0|P1|E2": "32", + "T1|P0|E2": "9", + "T1|P1|E2": "5", + "TNA|P0|E0": "28", + "TNA|P0|E1": "21", + "TNA|P0|E2": "20", + "TNA|P1|E0": "29", + "TNA|P1|E1": "7", + "TNA|P1|E2": "14" + }, + "n_rare_merged": "6" +} \ No newline at end of file diff --git a/models/cv5_sample/moe_llm/seed42/summary.json b/models/cv5_sample/moe_llm/seed42/summary.json new file mode 100644 index 0000000..08a5fe2 --- /dev/null +++ b/models/cv5_sample/moe_llm/seed42/summary.json @@ -0,0 +1,372 @@ +{ + "fold_results": [ + { + "fold": 0, + "best_params": { + "dropout": 0.19323529834582587, + "lr": 0.000980484886752915, + "weight_decay": 0.00014367509113739864, + "backbone_lr_ratio": 0.13093310334961422, + "moe_n_experts": 4, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 11, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.4858819540561683, + "rmse": 1.2189675771144073, + "mae": 0.5309677058494234, + "r2": 0.0906036692787614 + }, + "delivery": { + "n_samples": 58, + "mse": 0.4578579988449137, + "rmse": 0.6766520515338099, + "mae": 0.44371995239936074, + "r2": 0.41803485452290234 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7857142857142857, + "precision": 0.7322033898305085, + "recall": 0.7441197434069851, + "f1": 0.7375 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7380952380952381, + "precision": 0.7146208708708709, + "recall": 0.7547619047619047, + "f1": 0.7160330091364574 + }, + "toxic": { + "n_samples": 58, + "accuracy": 0.9827586206896551, + "precision": 0.9912280701754386, + "recall": 0.75, + "f1": 0.8289085545722714 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.2124931399005794, + "js_divergence": 0.0436771104854127 + } + } + }, + { + "fold": 1, + "best_params": { + "dropout": 0.19323529834582587, + "lr": 0.000980484886752915, + "weight_decay": 0.00014367509113739864, + "backbone_lr_ratio": 0.13093310334961422, + "moe_n_experts": 4, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 11, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.3521366695033162, + "rmse": 0.593411045990312, + "mae": 0.417994069290303, + "r2": -0.07530019609216088 + }, + "delivery": { + "n_samples": 61, + "mse": 0.8615718355352413, + "rmse": 0.9282089395902419, + "mae": 0.54733644703739, + "r2": 0.34785223029308054 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.6309523809523809, + "precision": 0.5417771883289124, + "recall": 0.5476190476190477, + "f1": 0.5421135924037278 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.5952380952380952, + "precision": 0.531971296677179, + "recall": 0.5323723895152467, + "f1": 0.5202020202020202 + }, + "toxic": { + "n_samples": 61, + "accuracy": 0.9836065573770492, + "precision": 0.9915254237288136, + "recall": 0.8333333333333333, + "f1": 0.8957264957264958 + }, + "biodist": { + "n_samples": 61, + "kl_divergence": 0.12922332899139097, + "js_divergence": 0.030354793549857463 + } + } + }, + { + "fold": 2, + "best_params": { + "dropout": 0.19323529834582587, + "lr": 0.000980484886752915, + "weight_decay": 0.00014367509113739864, + "backbone_lr_ratio": 0.13093310334961422, + "moe_n_experts": 4, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 11, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.4170957202870868, + "rmse": 0.6458294823613171, + "mae": 0.4985973424876907, + "r2": 0.1762434046684933 + }, + "delivery": { + "n_samples": 60, + "mse": 0.5854544655957109, + "rmse": 0.7651499628149445, + "mae": 0.49963968732239056, + "r2": 0.22065863304023947 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7380952380952381, + "precision": 0.6851851851851851, + "recall": 0.7199413489736071, + "f1": 0.693633952254642 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6666666666666666, + "precision": 0.613035584604212, + "recall": 0.6666666666666666, + "f1": 0.627436739505705 + }, + "toxic": { + "n_samples": 61, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.17218262970209255, + "js_divergence": 0.039882423974733884 + } + } + }, + { + "fold": 3, + "best_params": { + "dropout": 0.19323529834582587, + "lr": 0.000980484886752915, + "weight_decay": 0.00014367509113739864, + "backbone_lr_ratio": 0.13093310334961422, + "moe_n_experts": 4, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 11, + "test_metrics": { + "size": { + "n_samples": 83, + "mse": 1.773779594197195, + "rmse": 1.331833170557482, + "mae": 0.5606547438887408, + "r2": -0.003263258057283336 + }, + "delivery": { + "n_samples": 59, + "mse": 0.6480387215025862, + "rmse": 0.8050085226272988, + "mae": 0.5227914613403254, + "r2": 0.37002936863191194 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.75, + "precision": 0.6789023521026372, + "recall": 0.684017595307918, + "f1": 0.6813008130081302 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.6428571428571429, + "precision": 0.5700750584471516, + "recall": 0.5891402714932127, + "f1": 0.5736800630417652 + }, + "toxic": { + "n_samples": 60, + "accuracy": 0.9833333333333333, + "precision": 0.875, + "recall": 0.9912280701754386, + "f1": 0.9241466498103665 + }, + "biodist": { + "n_samples": 60, + "kl_divergence": 0.15399347135454697, + "js_divergence": 0.036819733414228055 + } + } + }, + { + "fold": 4, + "best_params": { + "dropout": 0.19323529834582587, + "lr": 0.000980484886752915, + "weight_decay": 0.00014367509113739864, + "backbone_lr_ratio": 0.13093310334961422, + "moe_n_experts": 4, + "moe_top_k": 2, + "moe_expert_hidden_mult": 2, + "llm_lora_r": 8, + "d_model": 256, + "num_heads": 8, + "n_attn_layers": 4, + "fusion_strategy": "attention", + "head_hidden_dim": 128, + "set_transformer_block": "sab" + }, + "epoch_mean": 11, + "test_metrics": { + "size": { + "n_samples": 84, + "mse": 0.6805742520206829, + "rmse": 0.8249692430755724, + "mae": 0.4822031688111435, + "r2": 0.08930711264607172 + }, + "delivery": { + "n_samples": 58, + "mse": 0.7669285932160084, + "rmse": 0.8757445935979328, + "mae": 0.567918715805843, + "r2": 0.2583927108955246 + }, + "pdi": { + "n_samples": 84, + "accuracy": 0.7619047619047619, + "precision": 0.7002652519893899, + "recall": 0.7214076246334311, + "f1": 0.7083333333333334 + }, + "ee": { + "n_samples": 84, + "accuracy": 0.7023809523809523, + "precision": 0.6327380952380953, + "recall": 0.6816239316239318, + "f1": 0.6431159420289855 + }, + "toxic": { + "n_samples": 59, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + }, + "biodist": { + "n_samples": 58, + "kl_divergence": 0.16534984510664474, + "js_divergence": 0.041883856181021084 + } + } + } + ], + "summary_stats": { + "size": { + "mse_mean": 0.9418936380128897, + "mse_std": 0.5795638860278585, + "rmse_mean": 0.9230021038198182, + "rmse_std": 0.29993458346292695, + "mae_mean": 0.4980834060654603, + "mae_std": 0.048297613647592774, + "r2_mean": 0.05551814648877644, + "r2_std": 0.0866209771152643 + }, + "delivery": { + "mse_mean": 0.6639703229388921, + "mse_std": 0.14039060447259435, + "rmse_mean": 0.8101528140328454, + "rmse_std": 0.08730830918963939, + "mae_mean": 0.5162812527810619, + "mae_std": 0.04293092010991082, + "r2_mean": 0.3229935594767318, + "r2_std": 0.07281328920995907 + }, + "pdi": { + "accuracy_mean": 0.7333333333333333, + "accuracy_std": 0.05355819942477139, + "precision_mean": 0.6676666734873267, + "precision_std": 0.06558804201410597, + "recall_mean": 0.6834210719881977, + "recall_std": 0.07057419243292298, + "f1_mean": 0.6725763381999668, + "f1_std": 0.06787298751865974 + }, + "ee": { + "accuracy_mean": 0.669047619047619, + "accuracy_std": 0.04902681019517619, + "precision_mean": 0.6124881811675018, + "precision_std": 0.061843813988713685, + "recall_mean": 0.6449130328121926, + "recall_std": 0.07702425935365928, + "f1_mean": 0.6160935547829867, + "f1_std": 0.06609215153794168 + }, + "toxic": { + "accuracy_mean": 0.9899397022800075, + "accuracy_std": 0.008218758400859814, + "precision_mean": 0.9715506987808504, + "precision_std": 0.04842922980294785, + "recall_mean": 0.9149122807017542, + "recall_std": 0.10407221017518521, + "f1_mean": 0.9297563400218267, + "f1_std": 0.06515845039804279 + }, + "biodist": { + "kl_divergence_mean": 0.16664848301105092, + "kl_divergence_std": 0.027183955851732975, + "js_divergence_mean": 0.03852358352105063, + "js_divergence_std": 0.004676087139485391 + } + } +} \ No newline at end of file diff --git a/scripts_run/run_cv5_sample.sh b/scripts_run/run_cv5_sample.sh new file mode 100644 index 0000000..83c6d9b --- /dev/null +++ b/scripts_run/run_cv5_sample.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# 样本级切分（StratifiedKFold）下 baseline vs MoE+LLM 对比实验。 +# 串行跑两个分支（24GB 单卡装不下两个 7B）。SSH 断连、GPU 抢占、OOM +# 都靠 --resume-dir 的折级断点续跑恢复，已完成的折不会重算。 +set -u + +GPU=${GPU:-0} +SEED=${SEED:-42} +ROOT=${ROOT:-models/cv5_sample} +FIX_HP=${FIX_HP:-models/final/best_params.json} +FIX_EM=${FIX_EM:-11} +BATCH=${BATCH:-8} +N_OUTER=${N_OUTER:-5} +N_INNER=${N_INNER:-3} + +MIN_FREE_MB=${MIN_FREE_MB:-12000} # 显存不够就等，不硬上 +WAIT_STEP=${WAIT_STEP:-60} +MAX_RETRY=${MAX_RETRY:-30} +RETRY_WAIT=${RETRY_WAIT:-120} +MIN_OK_SEC=${MIN_OK_SEC:-120} # 存活不足这么久且无进度 -> 判为配置错误，停止重试 + +mkdir -p "${ROOT}" +LOG="${ROOT}/run_seed${SEED}.log" +log() { echo "[$(date '+%F %T')] $*" >>"${LOG}"; } + +done_folds() { # $1 = 分支目录 + ls "$1"/outer_fold_*/test_metrics.json 2>/dev/null | wc -l +} + +wait_free() { + while :; do + local free + free=$(nvidia-smi --id="${GPU}" --query-gpu=memory.free \ + --format=csv,noheader,nounits 2>/dev/null || echo 0) + [ "${free}" -ge "${MIN_FREE_MB}" ] && break + log "GPU${GPU} 空闲 ${free}MiB < ${MIN_FREE_MB}MiB，${WAIT_STEP}s 后重试" + sleep "${WAIT_STEP}" + done +} + +# $1 = 分支名，其余参数为该分支特有的 CLI 开关 +run_variant() { + local name=$1; shift + local dir="${ROOT}/${name}/seed${SEED}" + mkdir -p "${dir}" + + local before after rc dt + before=$(done_folds "${dir}") + if [ "${before}" -ge "${N_OUTER}" ]; then + log "${name}: ${before}/${N_OUTER} 折已完成，整个分支跳过" + return 0 + fi + + local attempt + for attempt in $(seq 1 "${MAX_RETRY}"); do + wait_free + before=$(done_folds "${dir}") + log "${name} ATTEMPT ${attempt}/${MAX_RETRY}（已完成 ${before}/${N_OUTER} 折）" + local t0=${SECONDS} + + CUDA_VISIBLE_DEVICES=${GPU} TRANSFORMERS_OFFLINE=1 TOKENIZERS_PARALLELISM=false \ + PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \ + python -u -m lnp_ml.modeling.nested_cv_optuna \ + --input-path data/interim/internal.csv \ + --output-dir "${ROOT}" \ + --resume-dir "${dir}" \ + --seed "${SEED}" \ + --use-mpnn \ + --fix-hparams-json "${FIX_HP}" \ + --fix-epoch-mean "${FIX_EM}" \ + --n-outer-folds "${N_OUTER}" --n-inner-folds "${N_INNER}" \ + --batch-size "${BATCH}" --device cuda \ + ${@+"$@"} >>"${LOG}" 2>&1 + rc=$? + dt=$((SECONDS - t0)) + + if [ ${rc} -eq 0 ]; then + log "${name} DONE（本次用时 ${dt}s，共 $(done_folds "${dir}")/${N_OUTER} 折）" + return 0 + fi + + after=$(done_folds "${dir}") + if [ ${dt} -lt ${MIN_OK_SEC} ] && [ "${after}" -le "${before}" ]; then + log "${name} 仅存活 ${dt}s 且无进度（rc=${rc}），判为配置/代码错误而非抢占，停止重试" + log "${name} 请查看本日志末尾的 traceback" + return ${rc} + fi + log "${name} 运行 ${dt}s 后以 ${rc} 退出（进度 ${before}->${after}），${RETRY_WAIT}s 后续跑" + sleep "${RETRY_WAIT}" + done + + log "${name} 重试 ${MAX_RETRY} 次仍未完成，放弃" + return 1 +} + +log "===== 开始：样本级切分对比实验 seed=${SEED} gpu=${GPU} =====" + +run_variant baseline || { log "baseline 失败，不再继续 moe_llm"; exit 1; } + +run_variant moe_llm \ + --use-moe --moe-n-experts 4 --moe-top-k 2 \ + --reg-bypass off \ + --use-llm --use-rag --rag-top-k 4 \ + --use-soft-prompt --llm-use-qlora \ + --llm-model-path models/qwen2.5-7b-instruct \ + --no-llm-freeze \ + --llm-max-length 1536 \ + || { log "moe_llm 失败"; exit 1; } + +log "===== 全部完成，两个分支各 ${N_OUTER} 折 ====="