layer,module,loss,samples,damp,time 0,self_attn.kv_a_proj_with_mqa,0.0000064490,0.01000,1.355 0,self_attn.kv_b_proj,0.0000056584,0.01000,0.091 0,self_attn.o_proj,0.0000000040,0.01000,0.333 0,mlp.gate_proj,0.0000025663,0.01000,0.338 0,mlp.up_proj,0.0000024884,0.01000,0.334 0,mlp.down_proj,0.0000000106,0.01000,1.957 1,self_attn.kv_a_proj_with_mqa,0.0000033331,0.01000,0.334 1,self_attn.kv_b_proj,0.0000041922,0.01000,0.086 1,self_attn.o_proj,0.0000000088,0.01000,0.336 1,mlp.experts.0.gate_proj,0.0000138224,0.01000,0.337 1,mlp.experts.1.gate_proj,0.0000329910,0.01000,0.334 1,mlp.experts.2.gate_proj,0.0000141572,0.01000,0.333 1,mlp.experts.3.gate_proj,0.0000269833,0.01000,0.334 1,mlp.experts.4.gate_proj,0.0000267067,0.01000,0.334 1,mlp.experts.5.gate_proj,0.0000074419,0.01000,0.336 1,mlp.experts.6.gate_proj,0.0000075774,0.01000,0.339 1,mlp.experts.7.gate_proj,0.0000113019,0.01000,0.334 1,mlp.experts.8.gate_proj,0.0000196334,0.01000,0.335 1,mlp.experts.9.gate_proj,0.0000133080,0.01000,0.334 1,mlp.experts.10.gate_proj,0.0000111821,0.01000,0.336 1,mlp.experts.11.gate_proj,0.0000140752,0.01000,0.334 1,mlp.experts.12.gate_proj,0.0000054012,0.01000,0.334 1,mlp.experts.13.gate_proj,0.0000801727,0.01000,0.335 1,mlp.experts.14.gate_proj,0.0000119841,0.01000,0.334 1,mlp.experts.15.gate_proj,0.0000194801,0.01000,0.336 1,mlp.experts.16.gate_proj,0.0000146953,0.01000,0.335 1,mlp.experts.17.gate_proj,0.0000129339,0.01000,0.336 1,mlp.experts.18.gate_proj,0.0000088757,0.01000,0.336 1,mlp.experts.19.gate_proj,0.0000241038,0.01000,0.335 1,mlp.experts.20.gate_proj,0.0000233611,0.01000,0.336 1,mlp.experts.21.gate_proj,0.0000289465,0.01000,0.335 1,mlp.experts.22.gate_proj,0.0000221364,0.01000,0.335 1,mlp.experts.23.gate_proj,0.0000994787,0.01000,0.335 1,mlp.experts.24.gate_proj,0.0000164038,0.01000,0.333 1,mlp.experts.25.gate_proj,0.0000187588,0.01000,0.335 1,mlp.experts.26.gate_proj,0.0000378011,0.01000,0.334 1,mlp.experts.27.gate_proj,0.0000174131,0.01000,0.335 1,mlp.experts.28.gate_proj,0.0000136826,0.01000,0.338 1,mlp.experts.29.gate_proj,0.0000236969,0.01000,0.334 1,mlp.experts.30.gate_proj,0.0000223955,0.01000,0.335 1,mlp.experts.31.gate_proj,0.0000480106,0.01000,0.335 1,mlp.experts.32.gate_proj,0.0000059688,0.01000,0.335 1,mlp.experts.33.gate_proj,0.0005292354,0.01000,0.333 1,mlp.experts.34.gate_proj,0.0000142616,0.01000,0.333 1,mlp.experts.35.gate_proj,0.0000080994,0.01000,0.334 1,mlp.experts.36.gate_proj,0.0000081426,0.01000,0.335 1,mlp.experts.37.gate_proj,0.0000248639,0.01000,0.335 1,mlp.experts.38.gate_proj,0.0000314855,0.01000,0.334 1,mlp.experts.39.gate_proj,0.0000034727,0.01000,0.335 1,mlp.experts.40.gate_proj,0.0000229135,0.01000,0.334 1,mlp.experts.41.gate_proj,0.0000525688,0.01000,0.336 1,mlp.experts.42.gate_proj,0.0000320556,0.01000,0.335 1,mlp.experts.43.gate_proj,0.0001779415,0.01000,0.335 1,mlp.experts.44.gate_proj,0.0000135169,0.01000,0.336 1,mlp.experts.45.gate_proj,0.0000171119,0.01000,0.334 1,mlp.experts.46.gate_proj,0.0000079991,0.01000,0.334 1,mlp.experts.47.gate_proj,0.0000394319,0.01000,0.335 1,mlp.experts.48.gate_proj,0.0000975884,0.01000,0.335 1,mlp.experts.49.gate_proj,0.0000134076,0.01000,0.336 1,mlp.experts.50.gate_proj,0.0000296981,0.01000,0.336 1,mlp.experts.51.gate_proj,0.0000053538,0.01000,0.337 1,mlp.experts.52.gate_proj,0.0000074758,0.01000,0.336 1,mlp.experts.53.gate_proj,0.0000145542,0.01000,0.335 1,mlp.experts.54.gate_proj,0.0000185842,0.01000,0.335 1,mlp.experts.55.gate_proj,0.0000246100,0.01000,0.334 1,mlp.experts.56.gate_proj,0.0000228787,0.01000,0.335 1,mlp.experts.57.gate_proj,0.0000314578,0.01000,0.336 1,mlp.experts.58.gate_proj,0.0000205638,0.01000,0.336 1,mlp.experts.59.gate_proj,0.0000107298,0.01000,0.338 1,mlp.experts.60.gate_proj,0.0000145730,0.01000,0.334 1,mlp.experts.61.gate_proj,0.0000295511,0.01000,0.335 1,mlp.experts.62.gate_proj,0.0000183181,0.01000,0.338 1,mlp.experts.63.gate_proj,0.0000439796,0.01000,0.340 1,mlp.experts.0.up_proj,0.0000134747,0.01000,0.336 1,mlp.experts.1.up_proj,0.0000317115,0.01000,0.332 1,mlp.experts.2.up_proj,0.0000138673,0.01000,0.336 1,mlp.experts.3.up_proj,0.0000265529,0.01000,0.339 1,mlp.experts.4.up_proj,0.0000260976,0.01000,0.337 1,mlp.experts.5.up_proj,0.0000072686,0.01000,0.337 1,mlp.experts.6.up_proj,0.0000072241,0.01000,0.335 1,mlp.experts.7.up_proj,0.0000111837,0.01000,0.337 1,mlp.experts.8.up_proj,0.0000187104,0.01000,0.336 1,mlp.experts.9.up_proj,0.0000128802,0.01000,0.338 1,mlp.experts.10.up_proj,0.0000109376,0.01000,0.336 1,mlp.experts.11.up_proj,0.0000137012,0.01000,0.336 1,mlp.experts.12.up_proj,0.0000052257,0.01000,0.337 1,mlp.experts.13.up_proj,0.0000792764,0.01000,0.338 1,mlp.experts.14.up_proj,0.0000115214,0.01000,0.334 1,mlp.experts.15.up_proj,0.0000189583,0.01000,0.337 1,mlp.experts.16.up_proj,0.0000142498,0.01000,0.336 1,mlp.experts.17.up_proj,0.0000126231,0.01000,0.334 1,mlp.experts.18.up_proj,0.0000084671,0.01000,0.337 1,mlp.experts.19.up_proj,0.0000231639,0.01000,0.339 1,mlp.experts.20.up_proj,0.0000223713,0.01000,0.333 1,mlp.experts.21.up_proj,0.0000278180,0.01000,0.333 1,mlp.experts.22.up_proj,0.0000213987,0.01000,0.337 1,mlp.experts.23.up_proj,0.0000963849,0.01000,0.333 1,mlp.experts.24.up_proj,0.0000160182,0.01000,0.332 1,mlp.experts.25.up_proj,0.0000183133,0.01000,0.333 1,mlp.experts.26.up_proj,0.0000370522,0.01000,0.341 1,mlp.experts.27.up_proj,0.0000171972,0.01000,0.334 1,mlp.experts.28.up_proj,0.0000132752,0.01000,0.337 1,mlp.experts.29.up_proj,0.0000231780,0.01000,0.334 1,mlp.experts.30.up_proj,0.0000216792,0.01000,0.333 1,mlp.experts.31.up_proj,0.0000463384,0.01000,0.331 1,mlp.experts.32.up_proj,0.0000057735,0.01000,0.334 1,mlp.experts.33.up_proj,0.0005117715,0.01000,0.333 1,mlp.experts.34.up_proj,0.0000139147,0.01000,0.333 1,mlp.experts.35.up_proj,0.0000079576,0.01000,0.334 1,mlp.experts.36.up_proj,0.0000073608,0.01000,0.334 1,mlp.experts.37.up_proj,0.0000240782,0.01000,0.334 1,mlp.experts.38.up_proj,0.0000306801,0.01000,0.336 1,mlp.experts.39.up_proj,0.0000031212,0.01000,0.334 1,mlp.experts.40.up_proj,0.0000226471,0.01000,0.332 1,mlp.experts.41.up_proj,0.0000511516,0.01000,0.335 1,mlp.experts.42.up_proj,0.0000312890,0.01000,0.342 1,mlp.experts.43.up_proj,0.0001727548,0.01000,0.336 1,mlp.experts.44.up_proj,0.0000129826,0.01000,0.354 1,mlp.experts.45.up_proj,0.0000166234,0.01000,0.350 1,mlp.experts.46.up_proj,0.0000078196,0.01000,0.337 1,mlp.experts.47.up_proj,0.0000382809,0.01000,0.349 1,mlp.experts.48.up_proj,0.0000950560,0.01000,0.347 1,mlp.experts.49.up_proj,0.0000131813,0.01000,0.346 1,mlp.experts.50.up_proj,0.0000289254,0.01000,0.346 1,mlp.experts.51.up_proj,0.0000051807,0.01000,0.334 1,mlp.experts.52.up_proj,0.0000072958,0.01000,0.349 1,mlp.experts.53.up_proj,0.0000139992,0.01000,0.333 1,mlp.experts.54.up_proj,0.0000181619,0.01000,0.334 1,mlp.experts.55.up_proj,0.0000240575,0.01000,0.332 1,mlp.experts.56.up_proj,0.0000220965,0.01000,0.333 1,mlp.experts.57.up_proj,0.0000302341,0.01000,0.332 1,mlp.experts.58.up_proj,0.0000192096,0.01000,0.332 1,mlp.experts.59.up_proj,0.0000104804,0.01000,0.334 1,mlp.experts.60.up_proj,0.0000142660,0.01000,0.332 1,mlp.experts.61.up_proj,0.0000280595,0.01000,0.334 1,mlp.experts.62.up_proj,0.0000177259,0.01000,0.333 1,mlp.experts.63.up_proj,0.0000420921,0.01000,0.332 1,mlp.experts.0.down_proj,0.0000001586,0.01000,0.232 1,mlp.experts.1.down_proj,0.0000005779,0.01000,0.230 1,mlp.experts.2.down_proj,0.0000001655,0.01000,0.230 1,mlp.experts.3.down_proj,0.0000004176,0.01000,0.230 1,mlp.experts.4.down_proj,0.0000004408,0.01000,0.231 1,mlp.experts.5.down_proj,0.0000001156,0.01000,0.230 1,mlp.experts.6.down_proj,0.0000001217,0.01000,0.230 1,mlp.experts.7.down_proj,0.0000001839,0.01000,0.230 1,mlp.experts.8.down_proj,0.0000002527,0.01000,0.229 1,mlp.experts.9.down_proj,0.0000001577,0.01000,0.230 1,mlp.experts.10.down_proj,0.0000001676,0.01000,0.230 1,mlp.experts.11.down_proj,0.0000002188,0.01000,0.230 1,mlp.experts.12.down_proj,0.0000000776,0.01000,0.230 1,mlp.experts.13.down_proj,0.0000013795,0.01000,0.231 1,mlp.experts.14.down_proj,0.0000001495,0.01000,0.229 1,mlp.experts.15.down_proj,0.0000002939,0.01000,0.229 1,mlp.experts.16.down_proj,0.0000002329,0.01000,0.229 1,mlp.experts.17.down_proj,0.0000001987,0.01000,0.229 1,mlp.experts.18.down_proj,0.0000001226,0.01000,0.230 1,mlp.experts.19.down_proj,0.0000002881,0.01000,0.233 1,mlp.experts.20.down_proj,0.0000005023,0.01000,0.229 1,mlp.experts.21.down_proj,0.0000005430,0.01000,0.229 1,mlp.experts.22.down_proj,0.0000003531,0.01000,0.229 1,mlp.experts.23.down_proj,0.0000014909,0.01000,0.230 1,mlp.experts.24.down_proj,0.0000002269,0.01000,0.231 1,mlp.experts.25.down_proj,0.0000002344,0.01000,0.229 1,mlp.experts.26.down_proj,0.0000005645,0.01000,0.229 1,mlp.experts.27.down_proj,0.0000003162,0.01000,0.230 1,mlp.experts.28.down_proj,0.0000001815,0.01000,0.230 1,mlp.experts.29.down_proj,0.0000003082,0.01000,0.229 1,mlp.experts.30.down_proj,0.0000002782,0.01000,0.230 1,mlp.experts.31.down_proj,0.0000006378,0.01000,0.230 1,mlp.experts.32.down_proj,0.0000000855,0.01000,0.230 1,mlp.experts.33.down_proj,0.0000083772,0.01000,0.229 1,mlp.experts.34.down_proj,0.0000001850,0.01000,0.228 1,mlp.experts.35.down_proj,0.0000001450,0.01000,0.230 1,mlp.experts.36.down_proj,0.0000001225,0.01000,0.229 1,mlp.experts.37.down_proj,0.0000003711,0.01000,0.231 1,mlp.experts.38.down_proj,0.0000003907,0.01000,0.231 1,mlp.experts.39.down_proj,0.0000000465,0.01000,0.229 1,mlp.experts.40.down_proj,0.0000003100,0.01000,0.229 1,mlp.experts.41.down_proj,0.0000009000,0.01000,0.230 1,mlp.experts.42.down_proj,0.0000004819,0.01000,0.229 1,mlp.experts.43.down_proj,0.0000030850,0.01000,0.230 1,mlp.experts.44.down_proj,0.0000002229,0.01000,0.231 1,mlp.experts.45.down_proj,0.0000002458,0.01000,0.230 1,mlp.experts.46.down_proj,0.0000001241,0.01000,0.230 1,mlp.experts.47.down_proj,0.0000005470,0.01000,0.230 1,mlp.experts.48.down_proj,0.0000015493,0.01000,0.230 1,mlp.experts.49.down_proj,0.0000001680,0.01000,0.229 1,mlp.experts.50.down_proj,0.0000003876,0.01000,0.230 1,mlp.experts.51.down_proj,0.0000000969,0.01000,0.230 1,mlp.experts.52.down_proj,0.0000001393,0.01000,0.229 1,mlp.experts.53.down_proj,0.0000002270,0.01000,0.229 1,mlp.experts.54.down_proj,0.0000002688,0.01000,0.229 1,mlp.experts.55.down_proj,0.0000003004,0.01000,0.229 1,mlp.experts.56.down_proj,0.0000002700,0.01000,0.229 1,mlp.experts.57.down_proj,0.0000005705,0.01000,0.229 1,mlp.experts.58.down_proj,0.0000003383,0.01000,0.229 1,mlp.experts.59.down_proj,0.0000001402,0.01000,0.230 1,mlp.experts.60.down_proj,0.0000002293,0.01000,0.230 1,mlp.experts.61.down_proj,0.0000005384,0.01000,0.229 1,mlp.experts.62.down_proj,0.0000002985,0.01000,0.230 1,mlp.experts.63.down_proj,0.0000005700,0.01000,0.231 1,mlp.shared_experts.gate_proj,0.0000016316,0.01000,0.341 1,mlp.shared_experts.up_proj,0.0000016803,0.01000,0.343 1,mlp.shared_experts.down_proj,0.0000000073,0.01000,0.465 2,self_attn.kv_a_proj_with_mqa,0.0000063859,0.01000,0.346 2,self_attn.kv_b_proj,0.0000288171,0.01000,0.091 2,self_attn.o_proj,0.0000000211,0.01000,0.341 2,mlp.experts.0.gate_proj,0.0004882973,0.01000,0.344 2,mlp.experts.1.gate_proj,0.0000216334,0.01000,0.337 2,mlp.experts.2.gate_proj,0.0000650883,0.01000,0.335 2,mlp.experts.3.gate_proj,0.0000190994,0.01000,0.338 2,mlp.experts.4.gate_proj,0.0000848032,0.01000,0.337 2,mlp.experts.5.gate_proj,0.0000280946,0.01000,0.334 2,mlp.experts.6.gate_proj,0.0000140498,0.01000,0.337 2,mlp.experts.7.gate_proj,0.0000256152,0.01000,0.337 2,mlp.experts.8.gate_proj,0.0000179003,0.01000,0.336 2,mlp.experts.9.gate_proj,0.0000668981,0.01000,0.337 2,mlp.experts.10.gate_proj,0.0001067187,0.01000,0.336 2,mlp.experts.11.gate_proj,0.0000333447,0.01000,0.336 2,mlp.experts.12.gate_proj,0.0000988597,0.01000,0.342 2,mlp.experts.13.gate_proj,0.0000572988,0.01000,0.336 2,mlp.experts.14.gate_proj,0.0000402104,0.01000,0.336 2,mlp.experts.15.gate_proj,0.0000191694,0.01000,0.335 2,mlp.experts.16.gate_proj,0.0000280694,0.01000,0.338 2,mlp.experts.17.gate_proj,0.0000921568,0.01000,0.338 2,mlp.experts.18.gate_proj,0.0000686134,0.01000,0.338 2,mlp.experts.19.gate_proj,0.0000236760,0.01000,0.338 2,mlp.experts.20.gate_proj,0.0000285837,0.01000,0.338 2,mlp.experts.21.gate_proj,0.0000145600,0.01000,0.336 2,mlp.experts.22.gate_proj,0.0000299612,0.01000,0.336 2,mlp.experts.23.gate_proj,0.0000272245,0.01000,0.337 2,mlp.experts.24.gate_proj,0.0001159344,0.01000,0.334 2,mlp.experts.25.gate_proj,0.0000308290,0.01000,0.335 2,mlp.experts.26.gate_proj,0.0000244469,0.01000,0.336 2,mlp.experts.27.gate_proj,0.0000279665,0.01000,0.338 2,mlp.experts.28.gate_proj,0.0000106997,0.01000,0.335 2,mlp.experts.29.gate_proj,0.0001340178,0.01000,0.337 2,mlp.experts.30.gate_proj,0.0000258187,0.01000,0.336 2,mlp.experts.31.gate_proj,0.0000493494,0.01000,0.338 2,mlp.experts.32.gate_proj,0.0000447823,0.01000,0.337 2,mlp.experts.33.gate_proj,0.0000276711,0.01000,0.338 2,mlp.experts.34.gate_proj,0.0000359207,0.01000,0.337 2,mlp.experts.35.gate_proj,0.0000562520,0.01000,0.335 2,mlp.experts.36.gate_proj,0.0000074107,0.01000,0.337 2,mlp.experts.37.gate_proj,0.0000356515,0.01000,0.334 2,mlp.experts.38.gate_proj,0.0000290966,0.01000,0.337 2,mlp.experts.39.gate_proj,0.0000204948,0.01000,0.335 2,mlp.experts.40.gate_proj,0.0000347761,0.01000,0.335 2,mlp.experts.41.gate_proj,0.0000225329,0.01000,0.336 2,mlp.experts.42.gate_proj,0.0000161034,0.01000,0.334 2,mlp.experts.43.gate_proj,0.0002535853,0.01000,0.337 2,mlp.experts.44.gate_proj,0.0000338676,0.01000,0.337 2,mlp.experts.45.gate_proj,0.0000424037,0.01000,0.352 2,mlp.experts.46.gate_proj,0.0000207482,0.01000,0.338 2,mlp.experts.47.gate_proj,0.0000390705,0.01000,0.338 2,mlp.experts.48.gate_proj,0.0000444832,0.01000,0.338 2,mlp.experts.49.gate_proj,0.0000137165,0.01000,0.338 2,mlp.experts.50.gate_proj,0.0000211843,0.01000,0.336 2,mlp.experts.51.gate_proj,0.0000264544,0.01000,0.338 2,mlp.experts.52.gate_proj,0.0001309536,0.01000,0.339 2,mlp.experts.53.gate_proj,0.0000343864,0.01000,0.339 2,mlp.experts.54.gate_proj,0.0000230066,0.01000,0.337 2,mlp.experts.55.gate_proj,0.0008257809,0.01000,0.338 2,mlp.experts.56.gate_proj,0.0000376716,0.01000,0.340 2,mlp.experts.57.gate_proj,0.0000368094,0.01000,0.336 2,mlp.experts.58.gate_proj,0.0000331223,0.01000,0.336 2,mlp.experts.59.gate_proj,0.0002070982,0.01000,0.335 2,mlp.experts.60.gate_proj,0.0002285094,0.01000,0.334 2,mlp.experts.61.gate_proj,0.0000266585,0.01000,0.335 2,mlp.experts.62.gate_proj,0.0000202556,0.01000,0.345 2,mlp.experts.63.gate_proj,0.0000102986,0.01000,0.335 2,mlp.experts.0.up_proj,0.0004565689,0.01000,0.344 2,mlp.experts.1.up_proj,0.0000205314,0.01000,0.338 2,mlp.experts.2.up_proj,0.0000632733,0.01000,0.338 2,mlp.experts.3.up_proj,0.0000186641,0.01000,0.339 2,mlp.experts.4.up_proj,0.0000796430,0.01000,0.339 2,mlp.experts.5.up_proj,0.0000259691,0.01000,0.336 2,mlp.experts.6.up_proj,0.0000127396,0.01000,0.337 2,mlp.experts.7.up_proj,0.0000242339,0.01000,0.339 2,mlp.experts.8.up_proj,0.0000167166,0.01000,0.335 2,mlp.experts.9.up_proj,0.0000626668,0.01000,0.336 2,mlp.experts.10.up_proj,0.0001013379,0.01000,0.337 2,mlp.experts.11.up_proj,0.0000320722,0.01000,0.334 2,mlp.experts.12.up_proj,0.0000936462,0.01000,0.332 2,mlp.experts.13.up_proj,0.0000542817,0.01000,0.338 2,mlp.experts.14.up_proj,0.0000385316,0.01000,0.346 2,mlp.experts.15.up_proj,0.0000174390,0.01000,0.333 2,mlp.experts.16.up_proj,0.0000259538,0.01000,0.339 2,mlp.experts.17.up_proj,0.0000859277,0.01000,0.337 2,mlp.experts.18.up_proj,0.0000648600,0.01000,0.335 2,mlp.experts.19.up_proj,0.0000230545,0.01000,0.342 2,mlp.experts.20.up_proj,0.0000249533,0.01000,0.336 2,mlp.experts.21.up_proj,0.0000141916,0.01000,0.336 2,mlp.experts.22.up_proj,0.0000283198,0.01000,0.332 2,mlp.experts.23.up_proj,0.0000261105,0.01000,0.335 2,mlp.experts.24.up_proj,0.0001096811,0.01000,0.337 2,mlp.experts.25.up_proj,0.0000297949,0.01000,0.338 2,mlp.experts.26.up_proj,0.0000225712,0.01000,0.334 2,mlp.experts.27.up_proj,0.0000270574,0.01000,0.336 2,mlp.experts.28.up_proj,0.0000097890,0.01000,0.335 2,mlp.experts.29.up_proj,0.0001272398,0.01000,0.336 2,mlp.experts.30.up_proj,0.0000250040,0.01000,0.335 2,mlp.experts.31.up_proj,0.0000477167,0.01000,0.335 2,mlp.experts.32.up_proj,0.0000420593,0.01000,0.333 2,mlp.experts.33.up_proj,0.0000257297,0.01000,0.333 2,mlp.experts.34.up_proj,0.0000334203,0.01000,0.335 2,mlp.experts.35.up_proj,0.0000538488,0.01000,0.333 2,mlp.experts.36.up_proj,0.0000067655,0.01000,0.336 2,mlp.experts.37.up_proj,0.0000337915,0.01000,0.352 2,mlp.experts.38.up_proj,0.0000275251,0.01000,0.335 2,mlp.experts.39.up_proj,0.0000197820,0.01000,0.338 2,mlp.experts.40.up_proj,0.0000330846,0.01000,0.354 2,mlp.experts.41.up_proj,0.0000210140,0.01000,0.338 2,mlp.experts.42.up_proj,0.0000153605,0.01000,0.339 2,mlp.experts.43.up_proj,0.0002408286,0.01000,0.335 2,mlp.experts.44.up_proj,0.0000294574,0.01000,0.337 2,mlp.experts.45.up_proj,0.0000400956,0.01000,0.335 2,mlp.experts.46.up_proj,0.0000199201,0.01000,0.336 2,mlp.experts.47.up_proj,0.0000378540,0.01000,0.337 2,mlp.experts.48.up_proj,0.0000424378,0.01000,0.338 2,mlp.experts.49.up_proj,0.0000132184,0.01000,0.339 2,mlp.experts.50.up_proj,0.0000194475,0.01000,0.335 2,mlp.experts.51.up_proj,0.0000250796,0.01000,0.337 2,mlp.experts.52.up_proj,0.0001249852,0.01000,0.341 2,mlp.experts.53.up_proj,0.0000331131,0.01000,0.338 2,mlp.experts.54.up_proj,0.0000221962,0.01000,0.336 2,mlp.experts.55.up_proj,0.0007810339,0.01000,0.336 2,mlp.experts.56.up_proj,0.0000354236,0.01000,0.342 2,mlp.experts.57.up_proj,0.0000348676,0.01000,0.341 2,mlp.experts.58.up_proj,0.0000316740,0.01000,0.336 2,mlp.experts.59.up_proj,0.0002005359,0.01000,0.335 2,mlp.experts.60.up_proj,0.0002184925,0.01000,0.337 2,mlp.experts.61.up_proj,0.0000254400,0.01000,0.335 2,mlp.experts.62.up_proj,0.0000197809,0.01000,0.343 2,mlp.experts.63.up_proj,0.0000096613,0.01000,0.337 2,mlp.experts.0.down_proj,0.0000076869,0.01000,0.238 2,mlp.experts.1.down_proj,0.0000002755,0.01000,0.231 2,mlp.experts.2.down_proj,0.0000012067,0.01000,0.231 2,mlp.experts.3.down_proj,0.0000002954,0.01000,0.229 2,mlp.experts.4.down_proj,0.0000011967,0.01000,0.231 2,mlp.experts.5.down_proj,0.0000003921,0.01000,0.230 2,mlp.experts.6.down_proj,0.0000002448,0.01000,0.233 2,mlp.experts.7.down_proj,0.0000003772,0.01000,0.231 2,mlp.experts.8.down_proj,0.0000002849,0.01000,0.230 2,mlp.experts.9.down_proj,0.0000009636,0.01000,0.231 2,mlp.experts.10.down_proj,0.0000015770,0.01000,0.231 2,mlp.experts.11.down_proj,0.0000004988,0.01000,0.231 2,mlp.experts.12.down_proj,0.0000014385,0.01000,0.231 2,mlp.experts.13.down_proj,0.0000009136,0.01000,0.230 2,mlp.experts.14.down_proj,0.0000007613,0.01000,0.231 2,mlp.experts.15.down_proj,0.0000002678,0.01000,0.230 2,mlp.experts.16.down_proj,0.0000003686,0.01000,0.231 2,mlp.experts.17.down_proj,0.0000018265,0.01000,0.231 2,mlp.experts.18.down_proj,0.0000010628,0.01000,0.231 2,mlp.experts.19.down_proj,0.0000003814,0.01000,0.229 2,mlp.experts.20.down_proj,0.0000006630,0.01000,0.233 2,mlp.experts.21.down_proj,0.0000002534,0.01000,0.232 2,mlp.experts.22.down_proj,0.0000003969,0.01000,0.231 2,mlp.experts.23.down_proj,0.0000005476,0.01000,0.229 2,mlp.experts.24.down_proj,0.0000019297,0.01000,0.231 2,mlp.experts.25.down_proj,0.0000005258,0.01000,0.231 2,mlp.experts.26.down_proj,0.0000004767,0.01000,0.232 2,mlp.experts.27.down_proj,0.0000004447,0.01000,0.230 2,mlp.experts.28.down_proj,0.0000001737,0.01000,0.232 2,mlp.experts.29.down_proj,0.0000018961,0.01000,0.230 2,mlp.experts.30.down_proj,0.0000004280,0.01000,0.233 2,mlp.experts.31.down_proj,0.0000009651,0.01000,0.230 2,mlp.experts.32.down_proj,0.0000008431,0.01000,0.231 2,mlp.experts.33.down_proj,0.0000003990,0.01000,0.231 2,mlp.experts.34.down_proj,0.0000006550,0.01000,0.231 2,mlp.experts.35.down_proj,0.0000008158,0.01000,0.235 2,mlp.experts.36.down_proj,0.0000001018,0.01000,0.232 2,mlp.experts.37.down_proj,0.0000004680,0.01000,0.232 2,mlp.experts.38.down_proj,0.0000005032,0.01000,0.232 2,mlp.experts.39.down_proj,0.0000003208,0.01000,0.243 2,mlp.experts.40.down_proj,0.0000004443,0.01000,0.234 2,mlp.experts.41.down_proj,0.0000003272,0.01000,0.234 2,mlp.experts.42.down_proj,0.0000002768,0.01000,0.232 2,mlp.experts.43.down_proj,0.0000039598,0.01000,0.234 2,mlp.experts.44.down_proj,0.0000004313,0.01000,0.231 2,mlp.experts.45.down_proj,0.0000006503,0.01000,0.233 2,mlp.experts.46.down_proj,0.0000003172,0.01000,0.230 2,mlp.experts.47.down_proj,0.0000007513,0.01000,0.233 2,mlp.experts.48.down_proj,0.0000006500,0.01000,0.231 2,mlp.experts.49.down_proj,0.0000002718,0.01000,0.231 2,mlp.experts.50.down_proj,0.0000002936,0.01000,0.231 2,mlp.experts.51.down_proj,0.0000003775,0.01000,0.229 2,mlp.experts.52.down_proj,0.0000019938,0.01000,0.231 2,mlp.experts.53.down_proj,0.0000004778,0.01000,0.231 2,mlp.experts.54.down_proj,0.0000003507,0.01000,0.232 2,mlp.experts.55.down_proj,0.0000165309,0.01000,0.231 2,mlp.experts.56.down_proj,0.0000007384,0.01000,0.232 2,mlp.experts.57.down_proj,0.0000004889,0.01000,0.231 2,mlp.experts.58.down_proj,0.0000005845,0.01000,0.231 2,mlp.experts.59.down_proj,0.0000041776,0.01000,0.231 2,mlp.experts.60.down_proj,0.0000031836,0.01000,0.232 2,mlp.experts.61.down_proj,0.0000004335,0.01000,0.234 2,mlp.experts.62.down_proj,0.0000003241,0.01000,0.235 2,mlp.experts.63.down_proj,0.0000001823,0.01000,0.233 2,mlp.shared_experts.gate_proj,0.0000031586,0.01000,0.343 2,mlp.shared_experts.up_proj,0.0000027924,0.01000,0.338 2,mlp.shared_experts.down_proj,0.0000000182,0.01000,0.470 3,self_attn.kv_a_proj_with_mqa,0.0000062856,0.01000,0.336 3,self_attn.kv_b_proj,0.0000280021,0.01000,0.092 3,self_attn.o_proj,0.0000000465,0.01000,0.339 3,mlp.experts.0.gate_proj,0.0000224010,0.01000,0.346 3,mlp.experts.1.gate_proj,0.0000719327,0.01000,0.337 3,mlp.experts.2.gate_proj,0.0000430188,0.01000,0.339 3,mlp.experts.3.gate_proj,0.0000382138,0.01000,0.336 3,mlp.experts.4.gate_proj,0.0000739216,0.01000,0.336 3,mlp.experts.5.gate_proj,0.0000889537,0.01000,0.339 3,mlp.experts.6.gate_proj,0.0000574650,0.01000,0.340 3,mlp.experts.7.gate_proj,0.0002108694,0.01000,0.336 3,mlp.experts.8.gate_proj,0.0000191471,0.01000,0.348 3,mlp.experts.9.gate_proj,0.0000323485,0.01000,0.336 3,mlp.experts.10.gate_proj,0.0000425454,0.01000,0.339 3,mlp.experts.11.gate_proj,0.0000717053,0.01000,0.336 3,mlp.experts.12.gate_proj,0.0000252607,0.01000,0.335 3,mlp.experts.13.gate_proj,0.0002230101,0.01000,0.338 3,mlp.experts.14.gate_proj,0.0000237705,0.01000,0.334 3,mlp.experts.15.gate_proj,0.0000508050,0.01000,0.336 3,mlp.experts.16.gate_proj,0.0000117352,0.01000,0.340 3,mlp.experts.17.gate_proj,0.0004657147,0.01000,0.339 3,mlp.experts.18.gate_proj,0.0000487735,0.01000,0.338 3,mlp.experts.19.gate_proj,0.0005394298,0.01000,0.339 3,mlp.experts.20.gate_proj,0.0000768014,0.01000,0.338 3,mlp.experts.21.gate_proj,0.0000909190,0.01000,0.336 3,mlp.experts.22.gate_proj,0.0012625685,0.01000,0.337 3,mlp.experts.23.gate_proj,0.0000443051,0.01000,0.337 3,mlp.experts.24.gate_proj,0.0000791351,0.01000,0.336 3,mlp.experts.25.gate_proj,0.0000783219,0.01000,0.335 3,mlp.experts.26.gate_proj,0.0000881925,0.01000,0.335 3,mlp.experts.27.gate_proj,0.0000364255,0.01000,0.337 3,mlp.experts.28.gate_proj,0.0000549185,0.01000,0.336 3,mlp.experts.29.gate_proj,0.0000146615,0.01000,0.339 3,mlp.experts.30.gate_proj,0.0000376080,0.01000,0.338 3,mlp.experts.31.gate_proj,0.0001044271,0.01000,0.337 3,mlp.experts.32.gate_proj,0.0000859980,0.01000,0.336 3,mlp.experts.33.gate_proj,0.0000275955,0.01000,0.336 3,mlp.experts.34.gate_proj,0.0000175088,0.01000,0.335 3,mlp.experts.35.gate_proj,0.0000659999,0.01000,0.336 3,mlp.experts.36.gate_proj,0.0000444064,0.01000,0.338 3,mlp.experts.37.gate_proj,0.0000645101,0.01000,0.337 3,mlp.experts.38.gate_proj,0.0000428121,0.01000,0.336 3,mlp.experts.39.gate_proj,0.0000226405,0.01000,0.337 3,mlp.experts.40.gate_proj,0.0000311797,0.01000,0.337 3,mlp.experts.41.gate_proj,0.0000272209,0.01000,0.338 3,mlp.experts.42.gate_proj,0.0000529033,0.01000,0.338 3,mlp.experts.43.gate_proj,0.0000263760,0.01000,0.340 3,mlp.experts.44.gate_proj,0.0000602840,0.01000,0.340 3,mlp.experts.45.gate_proj,0.0000264851,0.01000,0.341 3,mlp.experts.46.gate_proj,0.0000576461,0.01000,0.342 3,mlp.experts.47.gate_proj,0.0005400921,0.01000,0.341 3,mlp.experts.48.gate_proj,0.0000332258,0.01000,0.338 3,mlp.experts.49.gate_proj,0.0000439870,0.01000,0.339 3,mlp.experts.50.gate_proj,0.0000417650,0.01000,0.342 3,mlp.experts.51.gate_proj,0.0000584113,0.01000,0.340 3,mlp.experts.52.gate_proj,0.0003374564,0.01000,0.338 3,mlp.experts.53.gate_proj,0.0000250174,0.01000,0.337 3,mlp.experts.54.gate_proj,0.0000562651,0.01000,0.340 3,mlp.experts.55.gate_proj,0.0021303240,0.01000,0.338 3,mlp.experts.56.gate_proj,0.0003048142,0.01000,0.335 3,mlp.experts.57.gate_proj,0.0000269881,0.01000,0.337 3,mlp.experts.58.gate_proj,0.0002556742,0.01000,0.337 3,mlp.experts.59.gate_proj,0.0000439790,0.01000,0.336 3,mlp.experts.60.gate_proj,0.0000715672,0.01000,0.342 3,mlp.experts.61.gate_proj,0.0000648578,0.01000,0.334 3,mlp.experts.62.gate_proj,0.0000229762,0.01000,0.338 3,mlp.experts.63.gate_proj,0.0014421499,0.01000,0.335 3,mlp.experts.0.up_proj,0.0000204089,0.01000,0.345 3,mlp.experts.1.up_proj,0.0000640137,0.01000,0.336 3,mlp.experts.2.up_proj,0.0000382539,0.01000,0.344 3,mlp.experts.3.up_proj,0.0000360431,0.01000,0.340 3,mlp.experts.4.up_proj,0.0000667703,0.01000,0.335 3,mlp.experts.5.up_proj,0.0000820863,0.01000,0.336 3,mlp.experts.6.up_proj,0.0000547912,0.01000,0.339 3,mlp.experts.7.up_proj,0.0001932120,0.01000,0.338 3,mlp.experts.8.up_proj,0.0000171521,0.01000,0.341 3,mlp.experts.9.up_proj,0.0000304036,0.01000,0.339 3,mlp.experts.10.up_proj,0.0000387071,0.01000,0.338 3,mlp.experts.11.up_proj,0.0000672446,0.01000,0.338 3,mlp.experts.12.up_proj,0.0000241520,0.01000,0.341 3,mlp.experts.13.up_proj,0.0002091236,0.01000,0.349 3,mlp.experts.14.up_proj,0.0000212755,0.01000,0.341 3,mlp.experts.15.up_proj,0.0000469989,0.01000,0.338 3,mlp.experts.16.up_proj,0.0000105331,0.01000,0.339 3,mlp.experts.17.up_proj,0.0004203206,0.01000,0.336 3,mlp.experts.18.up_proj,0.0000461904,0.01000,0.339 3,mlp.experts.19.up_proj,0.0005125245,0.01000,0.338 3,mlp.experts.20.up_proj,0.0000708924,0.01000,0.339 3,mlp.experts.21.up_proj,0.0000873081,0.01000,0.339 3,mlp.experts.22.up_proj,0.0011920595,0.01000,0.340 3,mlp.experts.23.up_proj,0.0000398680,0.01000,0.341 3,mlp.experts.24.up_proj,0.0000685075,0.01000,0.338 3,mlp.experts.25.up_proj,0.0000701989,0.01000,0.341 3,mlp.experts.26.up_proj,0.0000810513,0.01000,0.338 3,mlp.experts.27.up_proj,0.0000336555,0.01000,0.339 3,mlp.experts.28.up_proj,0.0000515037,0.01000,0.338 3,mlp.experts.29.up_proj,0.0000112390,0.01000,0.339 3,mlp.experts.30.up_proj,0.0000323827,0.01000,0.341 3,mlp.experts.31.up_proj,0.0000991167,0.01000,0.337 3,mlp.experts.32.up_proj,0.0000805259,0.01000,0.337 3,mlp.experts.33.up_proj,0.0000255184,0.01000,0.339 3,mlp.experts.34.up_proj,0.0000164113,0.01000,0.339 3,mlp.experts.35.up_proj,0.0000615980,0.01000,0.339 3,mlp.experts.36.up_proj,0.0000422566,0.01000,0.339 3,mlp.experts.37.up_proj,0.0000589175,0.01000,0.339 3,mlp.experts.38.up_proj,0.0000383607,0.01000,0.338 3,mlp.experts.39.up_proj,0.0000213365,0.01000,0.340 3,mlp.experts.40.up_proj,0.0000288352,0.01000,0.340 3,mlp.experts.41.up_proj,0.0000261583,0.01000,0.340 3,mlp.experts.42.up_proj,0.0000493719,0.01000,0.337 3,mlp.experts.43.up_proj,0.0000234434,0.01000,0.341 3,mlp.experts.44.up_proj,0.0000546449,0.01000,0.338 3,mlp.experts.45.up_proj,0.0000243944,0.01000,0.339 3,mlp.experts.46.up_proj,0.0000546725,0.01000,0.338 3,mlp.experts.47.up_proj,0.0004891342,0.01000,0.340 3,mlp.experts.48.up_proj,0.0000319087,0.01000,0.340 3,mlp.experts.49.up_proj,0.0000409786,0.01000,0.340 3,mlp.experts.50.up_proj,0.0000364086,0.01000,0.340 3,mlp.experts.51.up_proj,0.0000549639,0.01000,0.342 3,mlp.experts.52.up_proj,0.0003213668,0.01000,0.339 3,mlp.experts.53.up_proj,0.0000231189,0.01000,0.338 3,mlp.experts.54.up_proj,0.0000518950,0.01000,0.338 3,mlp.experts.55.up_proj,0.0019615291,0.01000,0.340 3,mlp.experts.56.up_proj,0.0002939510,0.01000,0.337 3,mlp.experts.57.up_proj,0.0000255828,0.01000,0.338 3,mlp.experts.58.up_proj,0.0002427297,0.01000,0.340 3,mlp.experts.59.up_proj,0.0000397841,0.01000,0.338 3,mlp.experts.60.up_proj,0.0000643540,0.01000,0.336 3,mlp.experts.61.up_proj,0.0000603840,0.01000,0.337 3,mlp.experts.62.up_proj,0.0000188675,0.01000,0.337 3,mlp.experts.63.up_proj,0.0013665627,0.01000,0.338 3,mlp.experts.0.down_proj,0.0000005156,0.01000,0.240 3,mlp.experts.1.down_proj,0.0000014186,0.01000,0.231 3,mlp.experts.2.down_proj,0.0000007998,0.01000,0.231 3,mlp.experts.3.down_proj,0.0000009790,0.01000,0.232 3,mlp.experts.4.down_proj,0.0000017898,0.01000,0.230 3,mlp.experts.5.down_proj,0.0000016927,0.01000,0.231 3,mlp.experts.6.down_proj,0.0000008086,0.01000,0.231 3,mlp.experts.7.down_proj,0.0000058175,0.01000,0.231 3,mlp.experts.8.down_proj,0.0000003621,0.01000,0.232 3,mlp.experts.9.down_proj,0.0000005965,0.01000,0.232 3,mlp.experts.10.down_proj,0.0000013792,0.01000,0.233 3,mlp.experts.11.down_proj,0.0000015003,0.01000,0.232 3,mlp.experts.12.down_proj,0.0000006136,0.01000,0.234 3,mlp.experts.13.down_proj,0.0000055190,0.01000,0.233 3,mlp.experts.14.down_proj,0.0000004355,0.01000,0.232 3,mlp.experts.15.down_proj,0.0000010496,0.01000,0.232 3,mlp.experts.16.down_proj,0.0000002278,0.01000,0.232 3,mlp.experts.17.down_proj,0.0000105909,0.01000,0.231 3,mlp.experts.18.down_proj,0.0000012242,0.01000,0.244 3,mlp.experts.19.down_proj,0.0000126797,0.01000,0.232 3,mlp.experts.20.down_proj,0.0000018164,0.01000,0.232 3,mlp.experts.21.down_proj,0.0000023308,0.01000,0.230 3,mlp.experts.22.down_proj,0.0000290450,0.01000,0.232 3,mlp.experts.23.down_proj,0.0000007847,0.01000,0.232 3,mlp.experts.24.down_proj,0.0000021416,0.01000,0.232 3,mlp.experts.25.down_proj,0.0000014606,0.01000,0.232 3,mlp.experts.26.down_proj,0.0000020907,0.01000,0.233 3,mlp.experts.27.down_proj,0.0000010022,0.01000,0.232 3,mlp.experts.28.down_proj,0.0000013411,0.01000,0.231 3,mlp.experts.29.down_proj,0.0000003628,0.01000,0.231 3,mlp.experts.30.down_proj,0.0000007744,0.01000,0.232 3,mlp.experts.31.down_proj,0.0000022514,0.01000,0.231 3,mlp.experts.32.down_proj,0.0000021036,0.01000,0.232 3,mlp.experts.33.down_proj,0.0000005700,0.01000,0.232 3,mlp.experts.34.down_proj,0.0000003300,0.01000,0.231 3,mlp.experts.35.down_proj,0.0000012966,0.01000,0.232 3,mlp.experts.36.down_proj,0.0000009851,0.01000,0.233 3,mlp.experts.37.down_proj,0.0000012346,0.01000,0.232 3,mlp.experts.38.down_proj,0.0000010447,0.01000,0.231 3,mlp.experts.39.down_proj,0.0000004119,0.01000,0.231 3,mlp.experts.40.down_proj,0.0000007085,0.01000,0.232 3,mlp.experts.41.down_proj,0.0000006253,0.01000,0.233 3,mlp.experts.42.down_proj,0.0000011865,0.01000,0.233 3,mlp.experts.43.down_proj,0.0000005582,0.01000,0.232 3,mlp.experts.44.down_proj,0.0000012203,0.01000,0.231 3,mlp.experts.45.down_proj,0.0000005861,0.01000,0.231 3,mlp.experts.46.down_proj,0.0000013445,0.01000,0.231 3,mlp.experts.47.down_proj,0.0000130848,0.01000,0.231 3,mlp.experts.48.down_proj,0.0000008113,0.01000,0.231 3,mlp.experts.49.down_proj,0.0000009088,0.01000,0.233 3,mlp.experts.50.down_proj,0.0000008966,0.01000,0.231 3,mlp.experts.51.down_proj,0.0000014172,0.01000,0.232 3,mlp.experts.52.down_proj,0.0000080625,0.01000,0.233 3,mlp.experts.53.down_proj,0.0000005782,0.01000,0.233 3,mlp.experts.54.down_proj,0.0000011214,0.01000,0.231 3,mlp.experts.55.down_proj,0.0000433806,0.01000,0.231 3,mlp.experts.56.down_proj,0.0000080772,0.01000,0.232 3,mlp.experts.57.down_proj,0.0000006515,0.01000,0.231 3,mlp.experts.58.down_proj,0.0000056043,0.01000,0.231 3,mlp.experts.59.down_proj,0.0000008638,0.01000,0.233 3,mlp.experts.60.down_proj,0.0000021099,0.01000,0.233 3,mlp.experts.61.down_proj,0.0000012787,0.01000,0.233 3,mlp.experts.62.down_proj,0.0000005706,0.01000,0.232 3,mlp.experts.63.down_proj,0.0000306517,0.01000,0.233 3,mlp.shared_experts.gate_proj,0.0000048772,0.01000,0.343 3,mlp.shared_experts.up_proj,0.0000040560,0.01000,0.335 3,mlp.shared_experts.down_proj,0.0000000347,0.01000,0.481 4,self_attn.kv_a_proj_with_mqa,0.0000064491,0.01000,0.339 4,self_attn.kv_b_proj,0.0000522236,0.01000,0.091 4,self_attn.o_proj,0.0000001596,0.01000,0.343 4,mlp.experts.0.gate_proj,0.0000880129,0.01000,0.344 4,mlp.experts.1.gate_proj,0.0003149201,0.01000,0.337 4,mlp.experts.2.gate_proj,0.0001737585,0.01000,0.337 4,mlp.experts.3.gate_proj,0.0000481687,0.01000,0.338 4,mlp.experts.4.gate_proj,0.0000785010,0.01000,0.338 4,mlp.experts.5.gate_proj,0.0000052457,0.01000,0.338 4,mlp.experts.6.gate_proj,0.0000932942,0.01000,0.340 4,mlp.experts.7.gate_proj,0.0000669404,0.01000,0.340 4,mlp.experts.8.gate_proj,0.0001458885,0.01000,0.337 4,mlp.experts.9.gate_proj,0.0000701433,0.01000,0.338 4,mlp.experts.10.gate_proj,0.0000385000,0.01000,0.338 4,mlp.experts.11.gate_proj,0.0001887999,0.01000,0.340 4,mlp.experts.12.gate_proj,0.0002997470,0.01000,0.337 4,mlp.experts.13.gate_proj,0.0000398844,0.01000,0.336 4,mlp.experts.14.gate_proj,0.0001479670,0.01000,0.338 4,mlp.experts.15.gate_proj,0.0000739569,0.01000,0.340 4,mlp.experts.16.gate_proj,0.0003853675,0.01000,0.336 4,mlp.experts.17.gate_proj,0.0002275038,0.01000,0.339 4,mlp.experts.18.gate_proj,0.0001084015,0.01000,0.338 4,mlp.experts.19.gate_proj,0.0000424513,0.01000,0.336 4,mlp.experts.20.gate_proj,0.0000911214,0.01000,0.337 4,mlp.experts.21.gate_proj,0.0000747319,0.01000,0.338 4,mlp.experts.22.gate_proj,0.0000582842,0.01000,0.336 4,mlp.experts.23.gate_proj,0.0000494376,0.01000,0.337 4,mlp.experts.24.gate_proj,0.0002060017,0.01000,0.338 4,mlp.experts.25.gate_proj,0.0007237995,0.01000,0.338 4,mlp.experts.26.gate_proj,0.0003004534,0.01000,0.339 4,mlp.experts.27.gate_proj,0.0001277410,0.01000,0.338 4,mlp.experts.28.gate_proj,0.0004333818,0.01000,0.339 4,mlp.experts.29.gate_proj,0.0000598808,0.01000,0.338 4,mlp.experts.30.gate_proj,0.0001271489,0.01000,0.338 4,mlp.experts.31.gate_proj,0.0001864688,0.01000,0.337 4,mlp.experts.32.gate_proj,0.0000409125,0.01000,0.339 4,mlp.experts.33.gate_proj,0.0000645783,0.01000,0.336 4,mlp.experts.34.gate_proj,0.0000269292,0.01000,0.337 4,mlp.experts.35.gate_proj,0.0000124582,0.01000,0.340 4,mlp.experts.36.gate_proj,0.0006174218,0.01000,0.337 4,mlp.experts.37.gate_proj,0.0000074260,0.01000,0.339 4,mlp.experts.38.gate_proj,0.0000605822,0.01000,0.337 4,mlp.experts.39.gate_proj,0.0000760773,0.01000,0.339 4,mlp.experts.40.gate_proj,0.0001773107,0.01000,0.349 4,mlp.experts.41.gate_proj,0.0000591517,0.01000,0.336 4,mlp.experts.42.gate_proj,0.0000414302,0.01000,0.338 4,mlp.experts.43.gate_proj,0.0001307210,0.01000,0.337 4,mlp.experts.44.gate_proj,0.0004245865,0.01000,0.336 4,mlp.experts.45.gate_proj,0.0012888554,0.01000,0.337 4,mlp.experts.46.gate_proj,0.0000838593,0.01000,0.338 4,mlp.experts.47.gate_proj,0.0007380713,0.01000,0.337 4,mlp.experts.48.gate_proj,0.0000581749,0.01000,0.339 4,mlp.experts.49.gate_proj,0.0001016460,0.01000,0.340 4,mlp.experts.50.gate_proj,0.0000649346,0.01000,0.340 4,mlp.experts.51.gate_proj,0.0000921887,0.01000,0.340 4,mlp.experts.52.gate_proj,0.0001194578,0.01000,0.338 4,mlp.experts.53.gate_proj,0.0002188191,0.01000,0.340 4,mlp.experts.54.gate_proj,0.0000543578,0.01000,0.341 4,mlp.experts.55.gate_proj,0.0000878435,0.01000,0.340 4,mlp.experts.56.gate_proj,0.0000072718,0.01000,0.338 4,mlp.experts.57.gate_proj,0.0000326582,0.01000,0.340 4,mlp.experts.58.gate_proj,0.0000401155,0.01000,0.341 4,mlp.experts.59.gate_proj,0.0001371479,0.01000,0.342 4,mlp.experts.60.gate_proj,0.0000495686,0.01000,0.340 4,mlp.experts.61.gate_proj,0.0000505637,0.01000,0.338 4,mlp.experts.62.gate_proj,0.0000754400,0.01000,0.337 4,mlp.experts.63.gate_proj,0.0000442727,0.01000,0.337 4,mlp.experts.0.up_proj,0.0000784944,0.01000,0.350 4,mlp.experts.1.up_proj,0.0002706105,0.01000,0.339 4,mlp.experts.2.up_proj,0.0001683954,0.01000,0.342 4,mlp.experts.3.up_proj,0.0000416929,0.01000,0.340 4,mlp.experts.4.up_proj,0.0000676838,0.01000,0.337 4,mlp.experts.5.up_proj,0.0000038821,0.01000,0.342 4,mlp.experts.6.up_proj,0.0000828936,0.01000,0.341 4,mlp.experts.7.up_proj,0.0000609259,0.01000,0.338 4,mlp.experts.8.up_proj,0.0001290825,0.01000,0.337 4,mlp.experts.9.up_proj,0.0000639604,0.01000,0.341 4,mlp.experts.10.up_proj,0.0000323407,0.01000,0.340 4,mlp.experts.11.up_proj,0.0001726773,0.01000,0.337 4,mlp.experts.12.up_proj,0.0002813703,0.01000,0.339 4,mlp.experts.13.up_proj,0.0000374505,0.01000,0.340 4,mlp.experts.14.up_proj,0.0001388287,0.01000,0.341 4,mlp.experts.15.up_proj,0.0000678113,0.01000,0.338 4,mlp.experts.16.up_proj,0.0003538772,0.01000,0.337 4,mlp.experts.17.up_proj,0.0002129474,0.01000,0.337 4,mlp.experts.18.up_proj,0.0000948685,0.01000,0.338 4,mlp.experts.19.up_proj,0.0000405889,0.01000,0.335 4,mlp.experts.20.up_proj,0.0000837404,0.01000,0.339 4,mlp.experts.21.up_proj,0.0000673825,0.01000,0.341 4,mlp.experts.22.up_proj,0.0000530856,0.01000,0.336 4,mlp.experts.23.up_proj,0.0000446697,0.01000,0.339 4,mlp.experts.24.up_proj,0.0001874154,0.01000,0.338 4,mlp.experts.25.up_proj,0.0006958137,0.01000,0.337 4,mlp.experts.26.up_proj,0.0002855102,0.01000,0.334 4,mlp.experts.27.up_proj,0.0001079048,0.01000,0.335 4,mlp.experts.28.up_proj,0.0004016921,0.01000,0.335 4,mlp.experts.29.up_proj,0.0000567595,0.01000,0.336 4,mlp.experts.30.up_proj,0.0001170858,0.01000,0.337 4,mlp.experts.31.up_proj,0.0001682284,0.01000,0.335 4,mlp.experts.32.up_proj,0.0000369172,0.01000,0.337 4,mlp.experts.33.up_proj,0.0000550088,0.01000,0.338 4,mlp.experts.34.up_proj,0.0000244185,0.01000,0.338 4,mlp.experts.35.up_proj,0.0000118997,0.01000,0.337 4,mlp.experts.36.up_proj,0.0005570951,0.01000,0.338 4,mlp.experts.37.up_proj,0.0000060418,0.01000,0.336 4,mlp.experts.38.up_proj,0.0000574630,0.01000,0.335 4,mlp.experts.39.up_proj,0.0000710898,0.01000,0.336 4,mlp.experts.40.up_proj,0.0001545183,0.01000,0.339 4,mlp.experts.41.up_proj,0.0000540982,0.01000,0.337 4,mlp.experts.42.up_proj,0.0000379924,0.01000,0.335 4,mlp.experts.43.up_proj,0.0001276538,0.01000,0.337 4,mlp.experts.44.up_proj,0.0003902515,0.01000,0.336 4,mlp.experts.45.up_proj,0.0010813456,0.01000,0.335 4,mlp.experts.46.up_proj,0.0000794973,0.01000,0.335 4,mlp.experts.47.up_proj,0.0006925762,0.01000,0.337 4,mlp.experts.48.up_proj,0.0000538615,0.01000,0.337 4,mlp.experts.49.up_proj,0.0000932606,0.01000,0.338 4,mlp.experts.50.up_proj,0.0000573111,0.01000,0.339 4,mlp.experts.51.up_proj,0.0000795136,0.01000,0.336 4,mlp.experts.52.up_proj,0.0001060513,0.01000,0.336 4,mlp.experts.53.up_proj,0.0002099942,0.01000,0.335 4,mlp.experts.54.up_proj,0.0000521946,0.01000,0.336 4,mlp.experts.55.up_proj,0.0000780968,0.01000,0.335 4,mlp.experts.56.up_proj,0.0000068690,0.01000,0.336 4,mlp.experts.57.up_proj,0.0000305120,0.01000,0.335 4,mlp.experts.58.up_proj,0.0000378431,0.01000,0.337 4,mlp.experts.59.up_proj,0.0001228369,0.01000,0.335 4,mlp.experts.60.up_proj,0.0000440387,0.01000,0.334 4,mlp.experts.61.up_proj,0.0000456231,0.01000,0.335 4,mlp.experts.62.up_proj,0.0000696902,0.01000,0.336 4,mlp.experts.63.up_proj,0.0000410524,0.01000,0.336 4,mlp.experts.0.down_proj,0.0000026273,0.01000,0.239 4,mlp.experts.1.down_proj,0.0000113496,0.01000,0.231 4,mlp.experts.2.down_proj,0.0000062606,0.01000,0.232 4,mlp.experts.3.down_proj,0.0000017458,0.01000,0.232 4,mlp.experts.4.down_proj,0.0000021266,0.01000,0.232 4,mlp.experts.5.down_proj,0.0000001676,0.01000,0.232 4,mlp.experts.6.down_proj,0.0000022850,0.01000,0.231 4,mlp.experts.7.down_proj,0.0000018449,0.01000,0.231 4,mlp.experts.8.down_proj,0.0000037656,0.01000,0.231 4,mlp.experts.9.down_proj,0.0000023532,0.01000,0.232 4,mlp.experts.10.down_proj,0.0000013082,0.01000,0.230 4,mlp.experts.11.down_proj,0.0000067497,0.01000,0.231 4,mlp.experts.12.down_proj,0.0000087695,0.01000,0.231 4,mlp.experts.13.down_proj,0.0000016208,0.01000,0.231 4,mlp.experts.14.down_proj,0.0000051099,0.01000,0.231 4,mlp.experts.15.down_proj,0.0000020235,0.01000,0.230 4,mlp.experts.16.down_proj,0.0000115667,0.01000,0.231 4,mlp.experts.17.down_proj,0.0000092861,0.01000,0.230 4,mlp.experts.18.down_proj,0.0000026226,0.01000,0.232 4,mlp.experts.19.down_proj,0.0000014993,0.01000,0.232 4,mlp.experts.20.down_proj,0.0000030400,0.01000,0.232 4,mlp.experts.21.down_proj,0.0000021369,0.01000,0.232 4,mlp.experts.22.down_proj,0.0000015017,0.01000,0.232 4,mlp.experts.23.down_proj,0.0000013734,0.01000,0.231 4,mlp.experts.24.down_proj,0.0000058453,0.01000,0.231 4,mlp.experts.25.down_proj,0.0000220241,0.01000,0.231 4,mlp.experts.26.down_proj,0.0000087222,0.01000,0.232 4,mlp.experts.27.down_proj,0.0000039607,0.01000,0.230 4,mlp.experts.28.down_proj,0.0000130309,0.01000,0.231 4,mlp.experts.29.down_proj,0.0000019560,0.01000,0.231 4,mlp.experts.30.down_proj,0.0000045605,0.01000,0.232 4,mlp.experts.31.down_proj,0.0000046624,0.01000,0.231 4,mlp.experts.32.down_proj,0.0000009593,0.01000,0.231 4,mlp.experts.33.down_proj,0.0000022010,0.01000,0.231 4,mlp.experts.34.down_proj,0.0000008949,0.01000,0.231 4,mlp.experts.35.down_proj,0.0000004263,0.01000,0.231 4,mlp.experts.36.down_proj,0.0000187507,0.01000,0.233 4,mlp.experts.37.down_proj,0.0000002146,0.01000,0.231 4,mlp.experts.38.down_proj,0.0000016373,0.01000,0.230 4,mlp.experts.39.down_proj,0.0000027005,0.01000,0.232 4,mlp.experts.40.down_proj,0.0000045254,0.01000,0.231 4,mlp.experts.41.down_proj,0.0000017327,0.01000,0.232 4,mlp.experts.42.down_proj,0.0000010291,0.01000,0.233 4,mlp.experts.43.down_proj,0.0000043965,0.01000,0.231 4,mlp.experts.44.down_proj,0.0000124785,0.01000,0.231 4,mlp.experts.45.down_proj,0.0000518039,0.01000,0.230 4,mlp.experts.46.down_proj,0.0000025471,0.01000,0.230 4,mlp.experts.47.down_proj,0.0000224466,0.01000,0.234 4,mlp.experts.48.down_proj,0.0000018535,0.01000,0.230 4,mlp.experts.49.down_proj,0.0000035613,0.01000,0.231 4,mlp.experts.50.down_proj,0.0000018160,0.01000,0.231 4,mlp.experts.51.down_proj,0.0000024355,0.01000,0.232 4,mlp.experts.52.down_proj,0.0000037065,0.01000,0.230 4,mlp.experts.53.down_proj,0.0000059108,0.01000,0.231 4,mlp.experts.54.down_proj,0.0000014779,0.01000,0.244 4,mlp.experts.55.down_proj,0.0000020922,0.01000,0.230 4,mlp.experts.56.down_proj,0.0000002111,0.01000,0.231 4,mlp.experts.57.down_proj,0.0000010791,0.01000,0.231 4,mlp.experts.58.down_proj,0.0000011827,0.01000,0.235 4,mlp.experts.59.down_proj,0.0000038928,0.01000,0.231 4,mlp.experts.60.down_proj,0.0000011573,0.01000,0.231 4,mlp.experts.61.down_proj,0.0000015775,0.01000,0.231 4,mlp.experts.62.down_proj,0.0000026009,0.01000,0.231 4,mlp.experts.63.down_proj,0.0000014150,0.01000,0.231 4,mlp.shared_experts.gate_proj,0.0000057179,0.01000,0.344 4,mlp.shared_experts.up_proj,0.0000047751,0.01000,0.337 4,mlp.shared_experts.down_proj,0.0000000898,0.01000,0.466 5,self_attn.kv_a_proj_with_mqa,0.0000044404,0.01000,0.360 5,self_attn.kv_b_proj,0.0000260715,0.01000,0.102 5,self_attn.o_proj,0.0000003283,0.01000,0.345 5,mlp.experts.0.gate_proj,0.0000126774,0.01000,0.344 5,mlp.experts.1.gate_proj,0.0003349141,0.01000,0.337 5,mlp.experts.2.gate_proj,0.0000507125,0.01000,0.335 5,mlp.experts.3.gate_proj,0.0000440676,0.01000,0.336 5,mlp.experts.4.gate_proj,0.0001826285,0.01000,0.338 5,mlp.experts.5.gate_proj,0.0000840405,0.01000,0.337 5,mlp.experts.6.gate_proj,0.0001963230,0.01000,0.336 5,mlp.experts.7.gate_proj,0.0000605058,0.01000,0.337 5,mlp.experts.8.gate_proj,0.0000626828,0.01000,0.336 5,mlp.experts.9.gate_proj,0.0000632047,0.01000,0.336 5,mlp.experts.10.gate_proj,0.0001521373,0.01000,0.337 5,mlp.experts.11.gate_proj,0.0000537804,0.01000,0.346 5,mlp.experts.12.gate_proj,0.0000308921,0.01000,0.337 5,mlp.experts.13.gate_proj,0.0001074952,0.01000,0.336 5,mlp.experts.14.gate_proj,0.0000759980,0.01000,0.338 5,mlp.experts.15.gate_proj,0.0000274196,0.01000,0.337 5,mlp.experts.16.gate_proj,0.0001464767,0.01000,0.338 5,mlp.experts.17.gate_proj,0.0006943177,0.01000,0.339 5,mlp.experts.18.gate_proj,0.0000482648,0.01000,0.338 5,mlp.experts.19.gate_proj,0.0001793719,0.01000,0.336 5,mlp.experts.20.gate_proj,0.0001289914,0.01000,0.336 5,mlp.experts.21.gate_proj,0.0001088849,0.01000,0.335 5,mlp.experts.22.gate_proj,0.0004232058,0.01000,0.336 5,mlp.experts.23.gate_proj,0.0001437048,0.01000,0.335 5,mlp.experts.24.gate_proj,0.0001853053,0.01000,0.335 5,mlp.experts.25.gate_proj,0.0000519443,0.01000,0.336 5,mlp.experts.26.gate_proj,0.0000562928,0.01000,0.336 5,mlp.experts.27.gate_proj,0.0000945795,0.01000,0.337 5,mlp.experts.28.gate_proj,0.0001922827,0.01000,0.335 5,mlp.experts.29.gate_proj,0.0000074220,0.01000,0.334 5,mlp.experts.30.gate_proj,0.0001409969,0.01000,0.337 5,mlp.experts.31.gate_proj,0.0000271662,0.01000,0.338 5,mlp.experts.32.gate_proj,0.0000558924,0.01000,0.338 5,mlp.experts.33.gate_proj,0.0003850062,0.01000,0.340 5,mlp.experts.34.gate_proj,0.0000988618,0.01000,0.338 5,mlp.experts.35.gate_proj,0.0001070334,0.01000,0.336 5,mlp.experts.36.gate_proj,0.0001048830,0.01000,0.339 5,mlp.experts.37.gate_proj,0.0002986337,0.01000,0.339 5,mlp.experts.38.gate_proj,0.0001193146,0.01000,0.336 5,mlp.experts.39.gate_proj,0.0000533247,0.01000,0.336 5,mlp.experts.40.gate_proj,0.0001012891,0.01000,0.338 5,mlp.experts.41.gate_proj,0.0001611413,0.01000,0.340 5,mlp.experts.42.gate_proj,0.0000615497,0.01000,0.336 5,mlp.experts.43.gate_proj,0.0000643991,0.01000,0.340 5,mlp.experts.44.gate_proj,0.0000374174,0.01000,0.340 5,mlp.experts.45.gate_proj,0.0000685240,0.01000,0.337 5,mlp.experts.46.gate_proj,0.0000452008,0.01000,0.335 5,mlp.experts.47.gate_proj,0.0000461359,0.01000,0.336 5,mlp.experts.48.gate_proj,0.0000275003,0.01000,0.334 5,mlp.experts.49.gate_proj,0.0000335643,0.01000,0.337 5,mlp.experts.50.gate_proj,0.0007988353,0.01000,0.337 5,mlp.experts.51.gate_proj,0.0001005217,0.01000,0.334 5,mlp.experts.52.gate_proj,0.0000750604,0.01000,0.338 5,mlp.experts.53.gate_proj,0.0000340494,0.01000,0.335 5,mlp.experts.54.gate_proj,0.0001060138,0.01000,0.336 5,mlp.experts.55.gate_proj,0.0000891037,0.01000,0.335 5,mlp.experts.56.gate_proj,0.0000943134,0.01000,0.335 5,mlp.experts.57.gate_proj,0.0001153023,0.01000,0.337 5,mlp.experts.58.gate_proj,0.0000113795,0.01000,0.337 5,mlp.experts.59.gate_proj,0.0001263849,0.01000,0.335 5,mlp.experts.60.gate_proj,0.0001512345,0.01000,0.336 5,mlp.experts.61.gate_proj,0.0001600688,0.01000,0.338 5,mlp.experts.62.gate_proj,0.0002516837,0.01000,0.338 5,mlp.experts.63.gate_proj,0.0000547726,0.01000,0.349 5,mlp.experts.0.up_proj,0.0000108506,0.01000,0.346 5,mlp.experts.1.up_proj,0.0003154433,0.01000,0.343 5,mlp.experts.2.up_proj,0.0000447350,0.01000,0.334 5,mlp.experts.3.up_proj,0.0000403940,0.01000,0.344 5,mlp.experts.4.up_proj,0.0001713343,0.01000,0.336 5,mlp.experts.5.up_proj,0.0000741397,0.01000,0.336 5,mlp.experts.6.up_proj,0.0001690799,0.01000,0.333 5,mlp.experts.7.up_proj,0.0000563860,0.01000,0.337 5,mlp.experts.8.up_proj,0.0000564697,0.01000,0.334 5,mlp.experts.9.up_proj,0.0000537284,0.01000,0.339 5,mlp.experts.10.up_proj,0.0001386444,0.01000,0.335 5,mlp.experts.11.up_proj,0.0000491026,0.01000,0.335 5,mlp.experts.12.up_proj,0.0000259409,0.01000,0.337 5,mlp.experts.13.up_proj,0.0001003921,0.01000,0.335 5,mlp.experts.14.up_proj,0.0000675609,0.01000,0.335 5,mlp.experts.15.up_proj,0.0000244440,0.01000,0.338 5,mlp.experts.16.up_proj,0.0001190411,0.01000,0.334 5,mlp.experts.17.up_proj,0.0006420701,0.01000,0.336 5,mlp.experts.18.up_proj,0.0000444427,0.01000,0.337 5,mlp.experts.19.up_proj,0.0001692141,0.01000,0.337 5,mlp.experts.20.up_proj,0.0001232472,0.01000,0.336 5,mlp.experts.21.up_proj,0.0000990912,0.01000,0.337 5,mlp.experts.22.up_proj,0.0003815960,0.01000,0.336 5,mlp.experts.23.up_proj,0.0001235288,0.01000,0.335 5,mlp.experts.24.up_proj,0.0001581498,0.01000,0.335 5,mlp.experts.25.up_proj,0.0000477026,0.01000,0.347 5,mlp.experts.26.up_proj,0.0000541607,0.01000,0.335 5,mlp.experts.27.up_proj,0.0000876348,0.01000,0.334 5,mlp.experts.28.up_proj,0.0001601070,0.01000,0.337 5,mlp.experts.29.up_proj,0.0000071774,0.01000,0.336 5,mlp.experts.30.up_proj,0.0001295177,0.01000,0.335 5,mlp.experts.31.up_proj,0.0000247963,0.01000,0.338 5,mlp.experts.32.up_proj,0.0000476597,0.01000,0.334 5,mlp.experts.33.up_proj,0.0003421584,0.01000,0.335 5,mlp.experts.34.up_proj,0.0000871587,0.01000,0.335 5,mlp.experts.35.up_proj,0.0001016611,0.01000,0.335 5,mlp.experts.36.up_proj,0.0000958408,0.01000,0.334 5,mlp.experts.37.up_proj,0.0002877046,0.01000,0.335 5,mlp.experts.38.up_proj,0.0001093506,0.01000,0.337 5,mlp.experts.39.up_proj,0.0000477152,0.01000,0.334 5,mlp.experts.40.up_proj,0.0000848283,0.01000,0.335 5,mlp.experts.41.up_proj,0.0001451987,0.01000,0.334 5,mlp.experts.42.up_proj,0.0000574499,0.01000,0.335 5,mlp.experts.43.up_proj,0.0000559250,0.01000,0.334 5,mlp.experts.44.up_proj,0.0000359262,0.01000,0.335 5,mlp.experts.45.up_proj,0.0000568747,0.01000,0.335 5,mlp.experts.46.up_proj,0.0000420849,0.01000,0.336 5,mlp.experts.47.up_proj,0.0000423566,0.01000,0.338 5,mlp.experts.48.up_proj,0.0000258336,0.01000,0.335 5,mlp.experts.49.up_proj,0.0000312921,0.01000,0.336 5,mlp.experts.50.up_proj,0.0007090344,0.01000,0.334 5,mlp.experts.51.up_proj,0.0000941600,0.01000,0.336 5,mlp.experts.52.up_proj,0.0000683596,0.01000,0.336 5,mlp.experts.53.up_proj,0.0000309670,0.01000,0.335 5,mlp.experts.54.up_proj,0.0000972838,0.01000,0.336 5,mlp.experts.55.up_proj,0.0000782180,0.01000,0.336 5,mlp.experts.56.up_proj,0.0000834895,0.01000,0.336 5,mlp.experts.57.up_proj,0.0001029865,0.01000,0.336 5,mlp.experts.58.up_proj,0.0000094430,0.01000,0.336 5,mlp.experts.59.up_proj,0.0001204324,0.01000,0.336 5,mlp.experts.60.up_proj,0.0001241442,0.01000,0.337 5,mlp.experts.61.up_proj,0.0001554521,0.01000,0.339 5,mlp.experts.62.up_proj,0.0002289542,0.01000,0.336 5,mlp.experts.63.up_proj,0.0000521385,0.01000,0.339 5,mlp.experts.0.down_proj,0.0000004815,0.01000,0.239 5,mlp.experts.1.down_proj,0.0000130705,0.01000,0.231 5,mlp.experts.2.down_proj,0.0000018769,0.01000,0.231 5,mlp.experts.3.down_proj,0.0000015967,0.01000,0.230 5,mlp.experts.4.down_proj,0.0000069909,0.01000,0.231 5,mlp.experts.5.down_proj,0.0000035890,0.01000,0.230 5,mlp.experts.6.down_proj,0.0000102290,0.01000,0.233 5,mlp.experts.7.down_proj,0.0000026002,0.01000,0.231 5,mlp.experts.8.down_proj,0.0000022264,0.01000,0.231 5,mlp.experts.9.down_proj,0.0000021442,0.01000,0.231 5,mlp.experts.10.down_proj,0.0000056509,0.01000,0.231 5,mlp.experts.11.down_proj,0.0000019493,0.01000,0.231 5,mlp.experts.12.down_proj,0.0000011351,0.01000,0.230 5,mlp.experts.13.down_proj,0.0000038550,0.01000,0.231 5,mlp.experts.14.down_proj,0.0000030199,0.01000,0.232 5,mlp.experts.15.down_proj,0.0000009107,0.01000,0.231 5,mlp.experts.16.down_proj,0.0000065003,0.01000,0.231 5,mlp.experts.17.down_proj,0.0000256425,0.01000,0.232 5,mlp.experts.18.down_proj,0.0000017501,0.01000,0.231 5,mlp.experts.19.down_proj,0.0000083179,0.01000,0.231 5,mlp.experts.20.down_proj,0.0000052449,0.01000,0.231 5,mlp.experts.21.down_proj,0.0000047682,0.01000,0.231 5,mlp.experts.22.down_proj,0.0000192354,0.01000,0.230 5,mlp.experts.23.down_proj,0.0000071546,0.01000,0.232 5,mlp.experts.24.down_proj,0.0000063007,0.01000,0.231 5,mlp.experts.25.down_proj,0.0000019749,0.01000,0.232 5,mlp.experts.26.down_proj,0.0000025291,0.01000,0.230 5,mlp.experts.27.down_proj,0.0000043214,0.01000,0.232 5,mlp.experts.28.down_proj,0.0000062576,0.01000,0.232 5,mlp.experts.29.down_proj,0.0000003131,0.01000,0.232 5,mlp.experts.30.down_proj,0.0000053544,0.01000,0.231 5,mlp.experts.31.down_proj,0.0000010898,0.01000,0.231 5,mlp.experts.32.down_proj,0.0000021308,0.01000,0.231 5,mlp.experts.33.down_proj,0.0000144850,0.01000,0.232 5,mlp.experts.34.down_proj,0.0000031793,0.01000,0.234 5,mlp.experts.35.down_proj,0.0000046258,0.01000,0.231 5,mlp.experts.36.down_proj,0.0000043179,0.01000,0.232 5,mlp.experts.37.down_proj,0.0000163760,0.01000,0.232 5,mlp.experts.38.down_proj,0.0000051578,0.01000,0.230 5,mlp.experts.39.down_proj,0.0000020135,0.01000,0.232 5,mlp.experts.40.down_proj,0.0000034954,0.01000,0.231 5,mlp.experts.41.down_proj,0.0000061777,0.01000,0.232 5,mlp.experts.42.down_proj,0.0000032143,0.01000,0.230 5,mlp.experts.43.down_proj,0.0000020196,0.01000,0.232 5,mlp.experts.44.down_proj,0.0000018704,0.01000,0.231 5,mlp.experts.45.down_proj,0.0000020722,0.01000,0.232 5,mlp.experts.46.down_proj,0.0000024226,0.01000,0.232 5,mlp.experts.47.down_proj,0.0000017034,0.01000,0.233 5,mlp.experts.48.down_proj,0.0000010550,0.01000,0.238 5,mlp.experts.49.down_proj,0.0000012678,0.01000,0.232 5,mlp.experts.50.down_proj,0.0000281743,0.01000,0.235 5,mlp.experts.51.down_proj,0.0000049216,0.01000,0.231 5,mlp.experts.52.down_proj,0.0000027622,0.01000,0.235 5,mlp.experts.53.down_proj,0.0000011751,0.01000,0.232 5,mlp.experts.54.down_proj,0.0000047804,0.01000,0.233 5,mlp.experts.55.down_proj,0.0000035350,0.01000,0.233 5,mlp.experts.56.down_proj,0.0000033111,0.01000,0.233 5,mlp.experts.57.down_proj,0.0000042235,0.01000,0.233 5,mlp.experts.58.down_proj,0.0000003751,0.01000,0.233 5,mlp.experts.59.down_proj,0.0000047423,0.01000,0.232 5,mlp.experts.60.down_proj,0.0000072244,0.01000,0.234 5,mlp.experts.61.down_proj,0.0000080331,0.01000,0.233 5,mlp.experts.62.down_proj,0.0000116477,0.01000,0.232 5,mlp.experts.63.down_proj,0.0000022321,0.01000,0.231 5,mlp.shared_experts.gate_proj,0.0000064897,0.01000,0.358 5,mlp.shared_experts.up_proj,0.0000058710,0.01000,0.334 5,mlp.shared_experts.down_proj,0.0000001654,0.01000,0.470 6,self_attn.kv_a_proj_with_mqa,0.0000061958,0.01000,0.342 6,self_attn.kv_b_proj,0.0000693557,0.01000,0.093 6,self_attn.o_proj,0.0000003823,0.01000,0.340 6,mlp.experts.0.gate_proj,0.0002231674,0.01000,0.342 6,mlp.experts.1.gate_proj,0.0000348217,0.01000,0.340 6,mlp.experts.2.gate_proj,0.0002135921,0.01000,0.338 6,mlp.experts.3.gate_proj,0.0000415375,0.01000,0.337 6,mlp.experts.4.gate_proj,0.0001401672,0.01000,0.338 6,mlp.experts.5.gate_proj,0.0002839881,0.01000,0.340 6,mlp.experts.6.gate_proj,0.0000429214,0.01000,0.339 6,mlp.experts.7.gate_proj,0.0000938609,0.01000,0.338 6,mlp.experts.8.gate_proj,0.0000686354,0.01000,0.338 6,mlp.experts.9.gate_proj,0.0002205856,0.01000,0.339 6,mlp.experts.10.gate_proj,0.0001263747,0.01000,0.334 6,mlp.experts.11.gate_proj,0.0000451436,0.01000,0.334 6,mlp.experts.12.gate_proj,0.0001964773,0.01000,0.335 6,mlp.experts.13.gate_proj,0.0001707857,0.01000,0.336 6,mlp.experts.14.gate_proj,0.0000352618,0.01000,0.335 6,mlp.experts.15.gate_proj,0.0000588271,0.01000,0.335 6,mlp.experts.16.gate_proj,0.0003770040,0.01000,0.335 6,mlp.experts.17.gate_proj,0.0003473993,0.01000,0.337 6,mlp.experts.18.gate_proj,0.0002695279,0.01000,0.334 6,mlp.experts.19.gate_proj,0.0001897146,0.01000,0.336 6,mlp.experts.20.gate_proj,0.0007937793,0.01000,0.335 6,mlp.experts.21.gate_proj,0.0000414264,0.01000,0.334 6,mlp.experts.22.gate_proj,0.0001078475,0.01000,0.334 6,mlp.experts.23.gate_proj,0.0006441174,0.01000,0.335 6,mlp.experts.24.gate_proj,0.0000142467,0.01000,0.335 6,mlp.experts.25.gate_proj,0.0001116431,0.01000,0.334 6,mlp.experts.26.gate_proj,0.0001094168,0.01000,0.336 6,mlp.experts.27.gate_proj,0.0001463612,0.01000,0.333 6,mlp.experts.28.gate_proj,0.0000298133,0.01000,0.334 6,mlp.experts.29.gate_proj,0.0000480452,0.01000,0.335 6,mlp.experts.30.gate_proj,0.0003675901,0.01000,0.335 6,mlp.experts.31.gate_proj,0.0024154217,0.01000,0.336 6,mlp.experts.32.gate_proj,0.0000382265,0.01000,0.336 6,mlp.experts.33.gate_proj,0.0002816407,0.01000,0.335 6,mlp.experts.34.gate_proj,0.0000605443,0.01000,0.335 6,mlp.experts.35.gate_proj,0.0000527371,0.01000,0.335 6,mlp.experts.36.gate_proj,0.0001151557,0.01000,0.336 6,mlp.experts.37.gate_proj,0.0001920071,0.01000,0.335 6,mlp.experts.38.gate_proj,0.0000961833,0.01000,0.334 6,mlp.experts.39.gate_proj,0.0000566069,0.01000,0.336 6,mlp.experts.40.gate_proj,0.0000803644,0.01000,0.335 6,mlp.experts.41.gate_proj,0.0000094507,0.01000,0.342 6,mlp.experts.42.gate_proj,0.0000439699,0.01000,0.338 6,mlp.experts.43.gate_proj,0.0000760435,0.01000,0.336 6,mlp.experts.44.gate_proj,0.0000843287,0.01000,0.338 6,mlp.experts.45.gate_proj,0.0001404246,0.01000,0.341 6,mlp.experts.46.gate_proj,0.0000535392,0.01000,0.341 6,mlp.experts.47.gate_proj,0.0000941738,0.01000,0.335 6,mlp.experts.48.gate_proj,0.0001025846,0.01000,0.343 6,mlp.experts.49.gate_proj,0.0001892856,0.01000,0.340 6,mlp.experts.50.gate_proj,0.0000309188,0.01000,0.338 6,mlp.experts.51.gate_proj,0.0003022056,0.01000,0.337 6,mlp.experts.52.gate_proj,0.0000277009,0.01000,0.340 6,mlp.experts.53.gate_proj,0.0000299572,0.01000,0.340 6,mlp.experts.54.gate_proj,0.0002219894,0.01000,0.338 6,mlp.experts.55.gate_proj,0.0000210125,0.01000,0.340 6,mlp.experts.56.gate_proj,0.0003234568,0.01000,0.339 6,mlp.experts.57.gate_proj,0.0001178615,0.01000,0.339 6,mlp.experts.58.gate_proj,0.0000298897,0.01000,0.337 6,mlp.experts.59.gate_proj,0.0001220849,0.01000,0.335 6,mlp.experts.60.gate_proj,0.0002075823,0.01000,0.334 6,mlp.experts.61.gate_proj,0.0000486848,0.01000,0.337 6,mlp.experts.62.gate_proj,0.0004347660,0.01000,0.344 6,mlp.experts.63.gate_proj,0.0000893484,0.01000,0.336 6,mlp.experts.0.up_proj,0.0001916814,0.01000,0.345 6,mlp.experts.1.up_proj,0.0000309945,0.01000,0.345 6,mlp.experts.2.up_proj,0.0001892755,0.01000,0.336 6,mlp.experts.3.up_proj,0.0000364479,0.01000,0.337 6,mlp.experts.4.up_proj,0.0001299899,0.01000,0.339 6,mlp.experts.5.up_proj,0.0002426821,0.01000,0.336 6,mlp.experts.6.up_proj,0.0000391621,0.01000,0.336 6,mlp.experts.7.up_proj,0.0000855343,0.01000,0.340 6,mlp.experts.8.up_proj,0.0000653668,0.01000,0.338 6,mlp.experts.9.up_proj,0.0002015717,0.01000,0.336 6,mlp.experts.10.up_proj,0.0001162699,0.01000,0.339 6,mlp.experts.11.up_proj,0.0000403202,0.01000,0.338 6,mlp.experts.12.up_proj,0.0001725949,0.01000,0.335 6,mlp.experts.13.up_proj,0.0001435338,0.01000,0.337 6,mlp.experts.14.up_proj,0.0000302464,0.01000,0.336 6,mlp.experts.15.up_proj,0.0000513429,0.01000,0.336 6,mlp.experts.16.up_proj,0.0003314343,0.01000,0.335 6,mlp.experts.17.up_proj,0.0003025032,0.01000,0.340 6,mlp.experts.18.up_proj,0.0002317045,0.01000,0.346 6,mlp.experts.19.up_proj,0.0001719793,0.01000,0.334 6,mlp.experts.20.up_proj,0.0007214255,0.01000,0.341 6,mlp.experts.21.up_proj,0.0000339066,0.01000,0.337 6,mlp.experts.22.up_proj,0.0000960124,0.01000,0.338 6,mlp.experts.23.up_proj,0.0005910881,0.01000,0.335 6,mlp.experts.24.up_proj,0.0000130807,0.01000,0.335 6,mlp.experts.25.up_proj,0.0001032818,0.01000,0.336 6,mlp.experts.26.up_proj,0.0000924098,0.01000,0.334 6,mlp.experts.27.up_proj,0.0001297426,0.01000,0.334 6,mlp.experts.28.up_proj,0.0000269315,0.01000,0.336 6,mlp.experts.29.up_proj,0.0000461316,0.01000,0.335 6,mlp.experts.30.up_proj,0.0003147583,0.01000,0.335 6,mlp.experts.31.up_proj,0.0020521307,0.01000,0.336 6,mlp.experts.32.up_proj,0.0000344233,0.01000,0.335 6,mlp.experts.33.up_proj,0.0002592310,0.01000,0.336 6,mlp.experts.34.up_proj,0.0000526459,0.01000,0.336 6,mlp.experts.35.up_proj,0.0000480302,0.01000,0.336 6,mlp.experts.36.up_proj,0.0000966762,0.01000,0.337 6,mlp.experts.37.up_proj,0.0001811895,0.01000,0.340 6,mlp.experts.38.up_proj,0.0000886506,0.01000,0.336 6,mlp.experts.39.up_proj,0.0000530284,0.01000,0.335 6,mlp.experts.40.up_proj,0.0000736522,0.01000,0.336 6,mlp.experts.41.up_proj,0.0000072078,0.01000,0.336 6,mlp.experts.42.up_proj,0.0000333755,0.01000,0.335 6,mlp.experts.43.up_proj,0.0000684947,0.01000,0.335 6,mlp.experts.44.up_proj,0.0000781988,0.01000,0.335 6,mlp.experts.45.up_proj,0.0001123306,0.01000,0.335 6,mlp.experts.46.up_proj,0.0000445745,0.01000,0.336 6,mlp.experts.47.up_proj,0.0000886278,0.01000,0.335 6,mlp.experts.48.up_proj,0.0000917601,0.01000,0.336 6,mlp.experts.49.up_proj,0.0001536806,0.01000,0.336 6,mlp.experts.50.up_proj,0.0000297728,0.01000,0.335 6,mlp.experts.51.up_proj,0.0002732759,0.01000,0.335 6,mlp.experts.52.up_proj,0.0000262324,0.01000,0.333 6,mlp.experts.53.up_proj,0.0000243892,0.01000,0.336 6,mlp.experts.54.up_proj,0.0002031446,0.01000,0.337 6,mlp.experts.55.up_proj,0.0000176092,0.01000,0.340 6,mlp.experts.56.up_proj,0.0002646401,0.01000,0.336 6,mlp.experts.57.up_proj,0.0001023831,0.01000,0.335 6,mlp.experts.58.up_proj,0.0000255157,0.01000,0.338 6,mlp.experts.59.up_proj,0.0001030650,0.01000,0.337 6,mlp.experts.60.up_proj,0.0001937561,0.01000,0.337 6,mlp.experts.61.up_proj,0.0000456820,0.01000,0.337 6,mlp.experts.62.up_proj,0.0003669569,0.01000,0.336 6,mlp.experts.63.up_proj,0.0000704550,0.01000,0.337 6,mlp.experts.0.down_proj,0.0000105094,0.01000,0.238 6,mlp.experts.1.down_proj,0.0000013629,0.01000,0.231 6,mlp.experts.2.down_proj,0.0000088732,0.01000,0.231 6,mlp.experts.3.down_proj,0.0000018564,0.01000,0.231 6,mlp.experts.4.down_proj,0.0000072820,0.01000,0.231 6,mlp.experts.5.down_proj,0.0000162292,0.01000,0.231 6,mlp.experts.6.down_proj,0.0000020673,0.01000,0.230 6,mlp.experts.7.down_proj,0.0000037941,0.01000,0.232 6,mlp.experts.8.down_proj,0.0000032677,0.01000,0.232 6,mlp.experts.9.down_proj,0.0000097738,0.01000,0.230 6,mlp.experts.10.down_proj,0.0000053912,0.01000,0.229 6,mlp.experts.11.down_proj,0.0000019366,0.01000,0.230 6,mlp.experts.12.down_proj,0.0000082064,0.01000,0.230 6,mlp.experts.13.down_proj,0.0000071877,0.01000,0.232 6,mlp.experts.14.down_proj,0.0000018176,0.01000,0.230 6,mlp.experts.15.down_proj,0.0000023882,0.01000,0.230 6,mlp.experts.16.down_proj,0.0000146930,0.01000,0.231 6,mlp.experts.17.down_proj,0.0000148149,0.01000,0.232 6,mlp.experts.18.down_proj,0.0000113663,0.01000,0.232 6,mlp.experts.19.down_proj,0.0000105131,0.01000,0.230 6,mlp.experts.20.down_proj,0.0000364486,0.01000,0.231 6,mlp.experts.21.down_proj,0.0000015989,0.01000,0.232 6,mlp.experts.22.down_proj,0.0000049154,0.01000,0.231 6,mlp.experts.23.down_proj,0.0000300006,0.01000,0.235 6,mlp.experts.24.down_proj,0.0000005937,0.01000,0.231 6,mlp.experts.25.down_proj,0.0000048912,0.01000,0.232 6,mlp.experts.26.down_proj,0.0000050335,0.01000,0.231 6,mlp.experts.27.down_proj,0.0000057687,0.01000,0.231 6,mlp.experts.28.down_proj,0.0000012443,0.01000,0.230 6,mlp.experts.29.down_proj,0.0000024813,0.01000,0.231 6,mlp.experts.30.down_proj,0.0000145899,0.01000,0.231 6,mlp.experts.31.down_proj,0.0001258810,0.01000,0.230 6,mlp.experts.32.down_proj,0.0000013953,0.01000,0.229 6,mlp.experts.33.down_proj,0.0000132218,0.01000,0.231 6,mlp.experts.34.down_proj,0.0000025141,0.01000,0.230 6,mlp.experts.35.down_proj,0.0000024140,0.01000,0.232 6,mlp.experts.36.down_proj,0.0000038048,0.01000,0.230 6,mlp.experts.37.down_proj,0.0000085131,0.01000,0.230 6,mlp.experts.38.down_proj,0.0000048223,0.01000,0.232 6,mlp.experts.39.down_proj,0.0000025203,0.01000,0.232 6,mlp.experts.40.down_proj,0.0000036128,0.01000,0.231 6,mlp.experts.41.down_proj,0.0000003246,0.01000,0.230 6,mlp.experts.42.down_proj,0.0000018167,0.01000,0.230 6,mlp.experts.43.down_proj,0.0000033722,0.01000,0.231 6,mlp.experts.44.down_proj,0.0000043743,0.01000,0.233 6,mlp.experts.45.down_proj,0.0000055054,0.01000,0.232 6,mlp.experts.46.down_proj,0.0000022628,0.01000,0.233 6,mlp.experts.47.down_proj,0.0000040291,0.01000,0.232 6,mlp.experts.48.down_proj,0.0000043879,0.01000,0.232 6,mlp.experts.49.down_proj,0.0000083801,0.01000,0.232 6,mlp.experts.50.down_proj,0.0000016526,0.01000,0.231 6,mlp.experts.51.down_proj,0.0000122155,0.01000,0.231 6,mlp.experts.52.down_proj,0.0000013680,0.01000,0.233 6,mlp.experts.53.down_proj,0.0000012280,0.01000,0.232 6,mlp.experts.54.down_proj,0.0000093458,0.01000,0.231 6,mlp.experts.55.down_proj,0.0000008217,0.01000,0.230 6,mlp.experts.56.down_proj,0.0000127852,0.01000,0.243 6,mlp.experts.57.down_proj,0.0000048346,0.01000,0.230 6,mlp.experts.58.down_proj,0.0000014136,0.01000,0.231 6,mlp.experts.59.down_proj,0.0000052774,0.01000,0.232 6,mlp.experts.60.down_proj,0.0000097641,0.01000,0.232 6,mlp.experts.61.down_proj,0.0000018393,0.01000,0.230 6,mlp.experts.62.down_proj,0.0000173594,0.01000,0.231 6,mlp.experts.63.down_proj,0.0000040332,0.01000,0.232 6,mlp.shared_experts.gate_proj,0.0000076834,0.01000,0.356 6,mlp.shared_experts.up_proj,0.0000069337,0.01000,0.338 6,mlp.shared_experts.down_proj,0.0000002029,0.01000,0.476 7,self_attn.kv_a_proj_with_mqa,0.0000083053,0.01000,0.341 7,self_attn.kv_b_proj,0.0000890166,0.01000,0.095 7,self_attn.o_proj,0.0000005243,0.01000,0.343 7,mlp.experts.0.gate_proj,0.0000950769,0.01000,0.342 7,mlp.experts.1.gate_proj,0.0000444120,0.01000,0.336 7,mlp.experts.2.gate_proj,0.0000746616,0.01000,0.340 7,mlp.experts.3.gate_proj,0.0000637907,0.01000,0.344 7,mlp.experts.4.gate_proj,0.0000180932,0.01000,0.336 7,mlp.experts.5.gate_proj,0.0000591329,0.01000,0.340 7,mlp.experts.6.gate_proj,0.0019131870,0.01000,0.337 7,mlp.experts.7.gate_proj,0.0000240576,0.01000,0.336 7,mlp.experts.8.gate_proj,0.0002039772,0.01000,0.337 7,mlp.experts.9.gate_proj,0.0002108428,0.01000,0.337 7,mlp.experts.10.gate_proj,0.0000635330,0.01000,0.335 7,mlp.experts.11.gate_proj,0.0000587949,0.01000,0.335 7,mlp.experts.12.gate_proj,0.0000311101,0.01000,0.338 7,mlp.experts.13.gate_proj,0.0002510357,0.01000,0.337 7,mlp.experts.14.gate_proj,0.0003263157,0.01000,0.336 7,mlp.experts.15.gate_proj,0.0001121083,0.01000,0.337 7,mlp.experts.16.gate_proj,0.0001654383,0.01000,0.336 7,mlp.experts.17.gate_proj,0.0001695142,0.01000,0.338 7,mlp.experts.18.gate_proj,0.0020171621,0.01000,0.338 7,mlp.experts.19.gate_proj,0.0000835677,0.01000,0.337 7,mlp.experts.20.gate_proj,0.0002505770,0.01000,0.337 7,mlp.experts.21.gate_proj,0.0005695998,0.01000,0.337 7,mlp.experts.22.gate_proj,0.0000340249,0.01000,0.337 7,mlp.experts.23.gate_proj,0.0000935608,0.01000,0.336 7,mlp.experts.24.gate_proj,0.0000890148,0.01000,0.337 7,mlp.experts.25.gate_proj,0.0001439821,0.01000,0.336 7,mlp.experts.26.gate_proj,0.0003941522,0.01000,0.337 7,mlp.experts.27.gate_proj,0.0002041065,0.01000,0.337 7,mlp.experts.28.gate_proj,0.0004875373,0.01000,0.335 7,mlp.experts.29.gate_proj,0.0003393916,0.01000,0.336 7,mlp.experts.30.gate_proj,0.0000459877,0.01000,0.338 7,mlp.experts.31.gate_proj,0.0000732319,0.01000,0.337 7,mlp.experts.32.gate_proj,0.0000324221,0.01000,0.333 7,mlp.experts.33.gate_proj,0.0000358309,0.01000,0.336 7,mlp.experts.34.gate_proj,0.0000363870,0.01000,0.334 7,mlp.experts.35.gate_proj,0.0001314069,0.01000,0.335 7,mlp.experts.36.gate_proj,0.0000408420,0.01000,0.335 7,mlp.experts.37.gate_proj,0.0001399937,0.01000,0.336 7,mlp.experts.38.gate_proj,0.0005502968,0.01000,0.333 7,mlp.experts.39.gate_proj,0.0000328972,0.01000,0.337 7,mlp.experts.40.gate_proj,0.0000502133,0.01000,0.336 7,mlp.experts.41.gate_proj,0.0000865980,0.01000,0.337 7,mlp.experts.42.gate_proj,0.0002006823,0.01000,0.335 7,mlp.experts.43.gate_proj,0.0012680893,0.01000,0.337 7,mlp.experts.44.gate_proj,0.0009877840,0.01000,0.335 7,mlp.experts.45.gate_proj,0.0000915995,0.01000,0.336 7,mlp.experts.46.gate_proj,0.0020769029,0.01000,0.337 7,mlp.experts.47.gate_proj,0.0002831783,0.01000,0.335 7,mlp.experts.48.gate_proj,0.0000698101,0.01000,0.334 7,mlp.experts.49.gate_proj,0.0001929802,0.01000,0.336 7,mlp.experts.50.gate_proj,0.0000808592,0.01000,0.336 7,mlp.experts.51.gate_proj,0.0000984216,0.01000,0.335 7,mlp.experts.52.gate_proj,0.0000098054,0.01000,0.338 7,mlp.experts.53.gate_proj,0.0001806673,0.01000,0.337 7,mlp.experts.54.gate_proj,0.0000351543,0.01000,0.337 7,mlp.experts.55.gate_proj,0.0000345846,0.01000,0.336 7,mlp.experts.56.gate_proj,0.0002139441,0.01000,0.337 7,mlp.experts.57.gate_proj,0.0001199084,0.01000,0.336 7,mlp.experts.58.gate_proj,0.0000611438,0.01000,0.336 7,mlp.experts.59.gate_proj,0.0000839659,0.01000,0.338 7,mlp.experts.60.gate_proj,0.0000415481,0.01000,0.342 7,mlp.experts.61.gate_proj,0.0000832499,0.01000,0.337 7,mlp.experts.62.gate_proj,0.0000619618,0.01000,0.336 7,mlp.experts.63.gate_proj,0.0002262443,0.01000,0.337 7,mlp.experts.0.up_proj,0.0000854415,0.01000,0.347 7,mlp.experts.1.up_proj,0.0000394682,0.01000,0.335 7,mlp.experts.2.up_proj,0.0000717825,0.01000,0.336 7,mlp.experts.3.up_proj,0.0000600220,0.01000,0.337 7,mlp.experts.4.up_proj,0.0000173530,0.01000,0.336 7,mlp.experts.5.up_proj,0.0000513817,0.01000,0.338 7,mlp.experts.6.up_proj,0.0017681010,0.01000,0.338 7,mlp.experts.7.up_proj,0.0000216980,0.01000,0.338 7,mlp.experts.8.up_proj,0.0001876980,0.01000,0.335 7,mlp.experts.9.up_proj,0.0001933700,0.01000,0.337 7,mlp.experts.10.up_proj,0.0000594928,0.01000,0.337 7,mlp.experts.11.up_proj,0.0000506769,0.01000,0.337 7,mlp.experts.12.up_proj,0.0000283896,0.01000,0.337 7,mlp.experts.13.up_proj,0.0002280080,0.01000,0.337 7,mlp.experts.14.up_proj,0.0002865611,0.01000,0.335 7,mlp.experts.15.up_proj,0.0000989944,0.01000,0.336 7,mlp.experts.16.up_proj,0.0001511237,0.01000,0.337 7,mlp.experts.17.up_proj,0.0001535332,0.01000,0.336 7,mlp.experts.18.up_proj,0.0015944997,0.01000,0.335 7,mlp.experts.19.up_proj,0.0000747849,0.01000,0.336 7,mlp.experts.20.up_proj,0.0002278975,0.01000,0.339 7,mlp.experts.21.up_proj,0.0004781310,0.01000,0.336 7,mlp.experts.22.up_proj,0.0000322413,0.01000,0.339 7,mlp.experts.23.up_proj,0.0000856242,0.01000,0.336 7,mlp.experts.24.up_proj,0.0000778292,0.01000,0.337 7,mlp.experts.25.up_proj,0.0001350587,0.01000,0.336 7,mlp.experts.26.up_proj,0.0003289687,0.01000,0.339 7,mlp.experts.27.up_proj,0.0001702346,0.01000,0.334 7,mlp.experts.28.up_proj,0.0004217212,0.01000,0.335 7,mlp.experts.29.up_proj,0.0003049429,0.01000,0.337 7,mlp.experts.30.up_proj,0.0000423985,0.01000,0.336 7,mlp.experts.31.up_proj,0.0000647613,0.01000,0.337 7,mlp.experts.32.up_proj,0.0000277387,0.01000,0.336 7,mlp.experts.33.up_proj,0.0000302290,0.01000,0.339 7,mlp.experts.34.up_proj,0.0000326985,0.01000,0.335 7,mlp.experts.35.up_proj,0.0001191234,0.01000,0.337 7,mlp.experts.36.up_proj,0.0000390434,0.01000,0.336 7,mlp.experts.37.up_proj,0.0001156091,0.01000,0.337 7,mlp.experts.38.up_proj,0.0004751553,0.01000,0.336 7,mlp.experts.39.up_proj,0.0000290173,0.01000,0.337 7,mlp.experts.40.up_proj,0.0000473659,0.01000,0.337 7,mlp.experts.41.up_proj,0.0000789337,0.01000,0.335 7,mlp.experts.42.up_proj,0.0001838800,0.01000,0.340 7,mlp.experts.43.up_proj,0.0011883210,0.01000,0.339 7,mlp.experts.44.up_proj,0.0007824005,0.01000,0.335 7,mlp.experts.45.up_proj,0.0000845031,0.01000,0.340 7,mlp.experts.46.up_proj,0.0016511741,0.01000,0.338 7,mlp.experts.47.up_proj,0.0002248003,0.01000,0.337 7,mlp.experts.48.up_proj,0.0000630255,0.01000,0.338 7,mlp.experts.49.up_proj,0.0001683860,0.01000,0.340 7,mlp.experts.50.up_proj,0.0000749864,0.01000,0.341 7,mlp.experts.51.up_proj,0.0000859247,0.01000,0.336 7,mlp.experts.52.up_proj,0.0000074303,0.01000,0.338 7,mlp.experts.53.up_proj,0.0001652069,0.01000,0.337 7,mlp.experts.54.up_proj,0.0000340573,0.01000,0.337 7,mlp.experts.55.up_proj,0.0000299546,0.01000,0.338 7,mlp.experts.56.up_proj,0.0001872689,0.01000,0.337 7,mlp.experts.57.up_proj,0.0001012957,0.01000,0.338 7,mlp.experts.58.up_proj,0.0000513590,0.01000,0.339 7,mlp.experts.59.up_proj,0.0000728837,0.01000,0.338 7,mlp.experts.60.up_proj,0.0000380854,0.01000,0.344 7,mlp.experts.61.up_proj,0.0000801046,0.01000,0.340 7,mlp.experts.62.up_proj,0.0000502170,0.01000,0.338 7,mlp.experts.63.up_proj,0.0001658654,0.01000,0.337 7,mlp.experts.0.down_proj,0.0000056788,0.01000,0.239 7,mlp.experts.1.down_proj,0.0000023230,0.01000,0.232 7,mlp.experts.2.down_proj,0.0000043901,0.01000,0.232 7,mlp.experts.3.down_proj,0.0000036811,0.01000,0.231 7,mlp.experts.4.down_proj,0.0000010919,0.01000,0.231 7,mlp.experts.5.down_proj,0.0000021732,0.01000,0.231 7,mlp.experts.6.down_proj,0.0001025670,0.01000,0.232 7,mlp.experts.7.down_proj,0.0000011812,0.01000,0.233 7,mlp.experts.8.down_proj,0.0000110781,0.01000,0.238 7,mlp.experts.9.down_proj,0.0000103959,0.01000,0.232 7,mlp.experts.10.down_proj,0.0000042371,0.01000,0.232 7,mlp.experts.11.down_proj,0.0000028560,0.01000,0.232 7,mlp.experts.12.down_proj,0.0000016707,0.01000,0.233 7,mlp.experts.13.down_proj,0.0000135892,0.01000,0.232 7,mlp.experts.14.down_proj,0.0000164666,0.01000,0.231 7,mlp.experts.15.down_proj,0.0000053245,0.01000,0.230 7,mlp.experts.16.down_proj,0.0000102551,0.01000,0.232 7,mlp.experts.17.down_proj,0.0000099852,0.01000,0.232 7,mlp.experts.18.down_proj,0.0001084684,0.01000,0.233 7,mlp.experts.19.down_proj,0.0000043298,0.01000,0.231 7,mlp.experts.20.down_proj,0.0000124049,0.01000,0.232 7,mlp.experts.21.down_proj,0.0000328957,0.01000,0.231 7,mlp.experts.22.down_proj,0.0000019907,0.01000,0.231 7,mlp.experts.23.down_proj,0.0000042795,0.01000,0.231 7,mlp.experts.24.down_proj,0.0000051758,0.01000,0.230 7,mlp.experts.25.down_proj,0.0000109984,0.01000,0.230 7,mlp.experts.26.down_proj,0.0000186900,0.01000,0.229 7,mlp.experts.27.down_proj,0.0000104180,0.01000,0.229 7,mlp.experts.28.down_proj,0.0000253212,0.01000,0.230 7,mlp.experts.29.down_proj,0.0000155006,0.01000,0.230 7,mlp.experts.30.down_proj,0.0000023391,0.01000,0.231 7,mlp.experts.31.down_proj,0.0000032661,0.01000,0.231 7,mlp.experts.32.down_proj,0.0000017082,0.01000,0.231 7,mlp.experts.33.down_proj,0.0000019820,0.01000,0.232 7,mlp.experts.34.down_proj,0.0000017431,0.01000,0.232 7,mlp.experts.35.down_proj,0.0000077195,0.01000,0.230 7,mlp.experts.36.down_proj,0.0000026418,0.01000,0.231 7,mlp.experts.37.down_proj,0.0000075260,0.01000,0.231 7,mlp.experts.38.down_proj,0.0000268309,0.01000,0.230 7,mlp.experts.39.down_proj,0.0000018807,0.01000,0.231 7,mlp.experts.40.down_proj,0.0000021344,0.01000,0.230 7,mlp.experts.41.down_proj,0.0000048877,0.01000,0.231 7,mlp.experts.42.down_proj,0.0000090475,0.01000,0.231 7,mlp.experts.43.down_proj,0.0000766883,0.01000,0.231 7,mlp.experts.44.down_proj,0.0000544532,0.01000,0.232 7,mlp.experts.45.down_proj,0.0000050000,0.01000,0.231 7,mlp.experts.46.down_proj,0.0001034070,0.01000,0.230 7,mlp.experts.47.down_proj,0.0000151453,0.01000,0.230 7,mlp.experts.48.down_proj,0.0000032409,0.01000,0.229 7,mlp.experts.49.down_proj,0.0000088286,0.01000,0.230 7,mlp.experts.50.down_proj,0.0000053330,0.01000,0.230 7,mlp.experts.51.down_proj,0.0000059152,0.01000,0.230 7,mlp.experts.52.down_proj,0.0000004565,0.01000,0.231 7,mlp.experts.53.down_proj,0.0000083489,0.01000,0.230 7,mlp.experts.54.down_proj,0.0000021335,0.01000,0.232 7,mlp.experts.55.down_proj,0.0000020351,0.01000,0.230 7,mlp.experts.56.down_proj,0.0000126213,0.01000,0.230 7,mlp.experts.57.down_proj,0.0000050701,0.01000,0.230 7,mlp.experts.58.down_proj,0.0000026500,0.01000,0.229 7,mlp.experts.59.down_proj,0.0000039802,0.01000,0.229 7,mlp.experts.60.down_proj,0.0000021311,0.01000,0.229 7,mlp.experts.61.down_proj,0.0000049379,0.01000,0.229 7,mlp.experts.62.down_proj,0.0000031169,0.01000,0.230 7,mlp.experts.63.down_proj,0.0000120402,0.01000,0.229 7,mlp.shared_experts.gate_proj,0.0000081737,0.01000,0.347 7,mlp.shared_experts.up_proj,0.0000074571,0.01000,0.335 7,mlp.shared_experts.down_proj,0.0000002794,0.01000,0.470 8,self_attn.kv_a_proj_with_mqa,0.0000052836,0.01000,0.337 8,self_attn.kv_b_proj,0.0000531933,0.01000,0.096 8,self_attn.o_proj,0.0000008782,0.01000,0.342 8,mlp.experts.0.gate_proj,0.0014290816,0.01000,0.346 8,mlp.experts.1.gate_proj,0.0000927910,0.01000,0.336 8,mlp.experts.2.gate_proj,0.0003639041,0.01000,0.335 8,mlp.experts.3.gate_proj,0.0002826653,0.01000,0.337 8,mlp.experts.4.gate_proj,0.0001353726,0.01000,0.336 8,mlp.experts.5.gate_proj,0.0001163357,0.01000,0.335 8,mlp.experts.6.gate_proj,0.0000461611,0.01000,0.337 8,mlp.experts.7.gate_proj,0.0037154963,0.01000,0.337 8,mlp.experts.8.gate_proj,0.0000138104,0.01000,0.335 8,mlp.experts.9.gate_proj,0.0001791377,0.01000,0.337 8,mlp.experts.10.gate_proj,0.0000569945,0.01000,0.336 8,mlp.experts.11.gate_proj,0.0001242297,0.01000,0.334 8,mlp.experts.12.gate_proj,0.0000498256,0.01000,0.336 8,mlp.experts.13.gate_proj,0.0001169843,0.01000,0.335 8,mlp.experts.14.gate_proj,0.0001257766,0.01000,0.335 8,mlp.experts.15.gate_proj,0.0000343507,0.01000,0.334 8,mlp.experts.16.gate_proj,0.0000672129,0.01000,0.344 8,mlp.experts.17.gate_proj,0.0000579832,0.01000,0.334 8,mlp.experts.18.gate_proj,0.0001069492,0.01000,0.335 8,mlp.experts.19.gate_proj,0.0003572751,0.01000,0.336 8,mlp.experts.20.gate_proj,0.0000724161,0.01000,0.333 8,mlp.experts.21.gate_proj,0.0001567761,0.01000,0.336 8,mlp.experts.22.gate_proj,0.0026652274,0.01000,0.337 8,mlp.experts.23.gate_proj,0.0006578725,0.01000,0.335 8,mlp.experts.24.gate_proj,0.0000404654,0.01000,0.335 8,mlp.experts.25.gate_proj,0.0009958353,0.01000,0.338 8,mlp.experts.26.gate_proj,0.0000114021,0.01000,0.338 8,mlp.experts.27.gate_proj,0.0002025188,0.01000,0.335 8,mlp.experts.28.gate_proj,0.0003132405,0.01000,0.341 8,mlp.experts.29.gate_proj,0.0001854903,0.01000,0.338 8,mlp.experts.30.gate_proj,0.0001521283,0.01000,0.334 8,mlp.experts.31.gate_proj,0.0000288091,0.01000,0.338 8,mlp.experts.32.gate_proj,0.0000289261,0.01000,0.337 8,mlp.experts.33.gate_proj,0.0002126768,0.01000,0.335 8,mlp.experts.34.gate_proj,0.0000328044,0.01000,0.337 8,mlp.experts.35.gate_proj,0.0000265852,0.01000,0.337 8,mlp.experts.36.gate_proj,0.0001032155,0.01000,0.335 8,mlp.experts.37.gate_proj,0.0047941085,0.01000,0.336 8,mlp.experts.38.gate_proj,0.0000375244,0.01000,0.337 8,mlp.experts.39.gate_proj,0.0001381485,0.01000,0.335 8,mlp.experts.40.gate_proj,0.0000399303,0.01000,0.334 8,mlp.experts.41.gate_proj,0.0000508060,0.01000,0.335 8,mlp.experts.42.gate_proj,0.0000983824,0.01000,0.334 8,mlp.experts.43.gate_proj,0.0016368285,0.01000,0.334 8,mlp.experts.44.gate_proj,0.0000150091,0.01000,0.333 8,mlp.experts.45.gate_proj,0.0008000424,0.01000,0.334 8,mlp.experts.46.gate_proj,0.0003242263,0.01000,0.336 8,mlp.experts.47.gate_proj,0.0000917793,0.01000,0.334 8,mlp.experts.48.gate_proj,0.0002874627,0.01000,0.335 8,mlp.experts.49.gate_proj,0.0016298828,0.01000,0.334 8,mlp.experts.50.gate_proj,0.0056801228,0.01000,0.335 8,mlp.experts.51.gate_proj,0.0000541640,0.01000,0.334 8,mlp.experts.52.gate_proj,0.0013352249,0.01000,0.334 8,mlp.experts.53.gate_proj,0.0001125187,0.01000,0.335 8,mlp.experts.54.gate_proj,0.0002287465,0.01000,0.334 8,mlp.experts.55.gate_proj,0.0001045781,0.01000,0.335 8,mlp.experts.56.gate_proj,0.0000244123,0.01000,0.336 8,mlp.experts.57.gate_proj,0.0000996568,0.01000,0.335 8,mlp.experts.58.gate_proj,0.0000526215,0.01000,0.335 8,mlp.experts.59.gate_proj,0.0000380093,0.01000,0.338 8,mlp.experts.60.gate_proj,0.0000971336,0.01000,0.335 8,mlp.experts.61.gate_proj,0.0001234986,0.01000,0.334 8,mlp.experts.62.gate_proj,0.0009946470,0.01000,0.335 8,mlp.experts.63.gate_proj,0.0001696552,0.01000,0.333 8,mlp.experts.0.up_proj,0.0012252564,0.01000,0.346 8,mlp.experts.1.up_proj,0.0000836938,0.01000,0.340 8,mlp.experts.2.up_proj,0.0002880118,0.01000,0.357 8,mlp.experts.3.up_proj,0.0002439706,0.01000,0.357 8,mlp.experts.4.up_proj,0.0001244062,0.01000,0.341 8,mlp.experts.5.up_proj,0.0001009387,0.01000,0.343 8,mlp.experts.6.up_proj,0.0000417972,0.01000,0.341 8,mlp.experts.7.up_proj,0.0030558592,0.01000,0.337 8,mlp.experts.8.up_proj,0.0000132634,0.01000,0.334 8,mlp.experts.9.up_proj,0.0001669951,0.01000,0.335 8,mlp.experts.10.up_proj,0.0000534727,0.01000,0.334 8,mlp.experts.11.up_proj,0.0001057684,0.01000,0.337 8,mlp.experts.12.up_proj,0.0000455325,0.01000,0.336 8,mlp.experts.13.up_proj,0.0001100523,0.01000,0.336 8,mlp.experts.14.up_proj,0.0001161121,0.01000,0.334 8,mlp.experts.15.up_proj,0.0000303110,0.01000,0.336 8,mlp.experts.16.up_proj,0.0000638988,0.01000,0.340 8,mlp.experts.17.up_proj,0.0000522863,0.01000,0.340 8,mlp.experts.18.up_proj,0.0000890294,0.01000,0.336 8,mlp.experts.19.up_proj,0.0002521402,0.01000,0.335 8,mlp.experts.20.up_proj,0.0000692425,0.01000,0.335 8,mlp.experts.21.up_proj,0.0001298416,0.01000,0.335 8,mlp.experts.22.up_proj,0.0022894716,0.01000,0.338 8,mlp.experts.23.up_proj,0.0005872603,0.01000,0.336 8,mlp.experts.24.up_proj,0.0000374679,0.01000,0.336 8,mlp.experts.25.up_proj,0.0008663661,0.01000,0.334 8,mlp.experts.26.up_proj,0.0000109544,0.01000,0.336 8,mlp.experts.27.up_proj,0.0001501115,0.01000,0.334 8,mlp.experts.28.up_proj,0.0002706176,0.01000,0.335 8,mlp.experts.29.up_proj,0.0001427833,0.01000,0.335 8,mlp.experts.30.up_proj,0.0001388821,0.01000,0.336 8,mlp.experts.31.up_proj,0.0000251057,0.01000,0.335 8,mlp.experts.32.up_proj,0.0000242391,0.01000,0.345 8,mlp.experts.33.up_proj,0.0001641914,0.01000,0.336 8,mlp.experts.34.up_proj,0.0000306850,0.01000,0.334 8,mlp.experts.35.up_proj,0.0000229464,0.01000,0.335 8,mlp.experts.36.up_proj,0.0000790523,0.01000,0.342 8,mlp.experts.37.up_proj,0.0039678827,0.01000,0.337 8,mlp.experts.38.up_proj,0.0000351290,0.01000,0.337 8,mlp.experts.39.up_proj,0.0001291171,0.01000,0.334 8,mlp.experts.40.up_proj,0.0000361265,0.01000,0.333 8,mlp.experts.41.up_proj,0.0000483357,0.01000,0.335 8,mlp.experts.42.up_proj,0.0000851340,0.01000,0.334 8,mlp.experts.43.up_proj,0.0014306203,0.01000,0.335 8,mlp.experts.44.up_proj,0.0000144713,0.01000,0.333 8,mlp.experts.45.up_proj,0.0006853200,0.01000,0.333 8,mlp.experts.46.up_proj,0.0002683570,0.01000,0.334 8,mlp.experts.47.up_proj,0.0000887905,0.01000,0.334 8,mlp.experts.48.up_proj,0.0002585816,0.01000,0.334 8,mlp.experts.49.up_proj,0.0013169649,0.01000,0.333 8,mlp.experts.50.up_proj,0.0052780042,0.01000,0.334 8,mlp.experts.51.up_proj,0.0000525634,0.01000,0.333 8,mlp.experts.52.up_proj,0.0011390585,0.01000,0.335 8,mlp.experts.53.up_proj,0.0000868589,0.01000,0.334 8,mlp.experts.54.up_proj,0.0002088956,0.01000,0.336 8,mlp.experts.55.up_proj,0.0000920326,0.01000,0.339 8,mlp.experts.56.up_proj,0.0000230663,0.01000,0.338 8,mlp.experts.57.up_proj,0.0000727161,0.01000,0.337 8,mlp.experts.58.up_proj,0.0000496177,0.01000,0.339 8,mlp.experts.59.up_proj,0.0000349437,0.01000,0.340 8,mlp.experts.60.up_proj,0.0000930415,0.01000,0.337 8,mlp.experts.61.up_proj,0.0001066082,0.01000,0.336 8,mlp.experts.62.up_proj,0.0008418477,0.01000,0.341 8,mlp.experts.63.up_proj,0.0001496476,0.01000,0.353 8,mlp.experts.0.down_proj,0.0000977840,0.01000,0.251 8,mlp.experts.1.down_proj,0.0000052128,0.01000,0.237 8,mlp.experts.2.down_proj,0.0000236639,0.01000,0.238 8,mlp.experts.3.down_proj,0.0000143241,0.01000,0.236 8,mlp.experts.4.down_proj,0.0000078215,0.01000,0.238 8,mlp.experts.5.down_proj,0.0000084518,0.01000,0.236 8,mlp.experts.6.down_proj,0.0000028357,0.01000,0.240 8,mlp.experts.7.down_proj,0.0002047798,0.01000,0.237 8,mlp.experts.8.down_proj,0.0000009188,0.01000,0.242 8,mlp.experts.9.down_proj,0.0000081064,0.01000,0.242 8,mlp.experts.10.down_proj,0.0000035437,0.01000,0.235 8,mlp.experts.11.down_proj,0.0000089316,0.01000,0.241 8,mlp.experts.12.down_proj,0.0000034452,0.01000,0.238 8,mlp.experts.13.down_proj,0.0000097896,0.01000,0.239 8,mlp.experts.14.down_proj,0.0000090663,0.01000,0.237 8,mlp.experts.15.down_proj,0.0000021367,0.01000,0.240 8,mlp.experts.16.down_proj,0.0000047332,0.01000,0.237 8,mlp.experts.17.down_proj,0.0000032738,0.01000,0.247 8,mlp.experts.18.down_proj,0.0000062294,0.01000,0.237 8,mlp.experts.19.down_proj,0.0000238667,0.01000,0.239 8,mlp.experts.20.down_proj,0.0000046388,0.01000,0.238 8,mlp.experts.21.down_proj,0.0000087069,0.01000,0.240 8,mlp.experts.22.down_proj,0.0001777561,0.01000,0.241 8,mlp.experts.23.down_proj,0.0000322171,0.01000,0.237 8,mlp.experts.24.down_proj,0.0000021343,0.01000,0.240 8,mlp.experts.25.down_proj,0.0000604723,0.01000,0.236 8,mlp.experts.26.down_proj,0.0000007905,0.01000,0.238 8,mlp.experts.27.down_proj,0.0000134959,0.01000,0.247 8,mlp.experts.28.down_proj,0.0000180445,0.01000,0.242 8,mlp.experts.29.down_proj,0.0000110619,0.01000,0.240 8,mlp.experts.30.down_proj,0.0000117088,0.01000,0.241 8,mlp.experts.31.down_proj,0.0000014484,0.01000,0.246 8,mlp.experts.32.down_proj,0.0000019772,0.01000,0.240 8,mlp.experts.33.down_proj,0.0000119470,0.01000,0.241 8,mlp.experts.34.down_proj,0.0000023523,0.01000,0.237 8,mlp.experts.35.down_proj,0.0000014828,0.01000,0.239 8,mlp.experts.36.down_proj,0.0000065728,0.01000,0.239 8,mlp.experts.37.down_proj,0.0003456675,0.01000,0.242 8,mlp.experts.38.down_proj,0.0000020578,0.01000,0.239 8,mlp.experts.39.down_proj,0.0000102384,0.01000,0.241 8,mlp.experts.40.down_proj,0.0000015993,0.01000,0.245 8,mlp.experts.41.down_proj,0.0000033451,0.01000,0.237 8,mlp.experts.42.down_proj,0.0000066810,0.01000,0.244 8,mlp.experts.43.down_proj,0.0000880646,0.01000,0.241 8,mlp.experts.44.down_proj,0.0000010688,0.01000,0.238 8,mlp.experts.45.down_proj,0.0000526969,0.01000,0.243 8,mlp.experts.46.down_proj,0.0000209406,0.01000,0.240 8,mlp.experts.47.down_proj,0.0000073893,0.01000,0.238 8,mlp.experts.48.down_proj,0.0000183681,0.01000,0.231 8,mlp.experts.49.down_proj,0.0001053119,0.01000,0.231 8,mlp.experts.50.down_proj,0.0004111830,0.01000,0.233 8,mlp.experts.51.down_proj,0.0000031727,0.01000,0.240 8,mlp.experts.52.down_proj,0.0000732623,0.01000,0.238 8,mlp.experts.53.down_proj,0.0000063138,0.01000,0.247 8,mlp.experts.54.down_proj,0.0000131097,0.01000,0.239 8,mlp.experts.55.down_proj,0.0000070097,0.01000,0.236 8,mlp.experts.56.down_proj,0.0000019343,0.01000,0.240 8,mlp.experts.57.down_proj,0.0000065059,0.01000,0.238 8,mlp.experts.58.down_proj,0.0000037701,0.01000,0.239 8,mlp.experts.59.down_proj,0.0000021110,0.01000,0.240 8,mlp.experts.60.down_proj,0.0000061997,0.01000,0.241 8,mlp.experts.61.down_proj,0.0000087957,0.01000,0.234 8,mlp.experts.62.down_proj,0.0000619547,0.01000,0.241 8,mlp.experts.63.down_proj,0.0000111556,0.01000,0.242 8,mlp.shared_experts.gate_proj,0.0000085991,0.01000,0.343 8,mlp.shared_experts.up_proj,0.0000078830,0.01000,0.334 8,mlp.shared_experts.down_proj,0.0000003382,0.01000,0.469 9,self_attn.kv_a_proj_with_mqa,0.0000072758,0.01000,0.337 9,self_attn.kv_b_proj,0.0000691758,0.01000,0.093 9,self_attn.o_proj,0.0000010000,0.01000,0.345 9,mlp.experts.0.gate_proj,0.0000622814,0.01000,0.344 9,mlp.experts.1.gate_proj,0.0013673229,0.01000,0.334 9,mlp.experts.2.gate_proj,0.0001388797,0.01000,0.335 9,mlp.experts.3.gate_proj,0.0000994763,0.01000,0.336 9,mlp.experts.4.gate_proj,0.0000239125,0.01000,0.336 9,mlp.experts.5.gate_proj,0.0002038196,0.01000,0.336 9,mlp.experts.6.gate_proj,0.0000447610,0.01000,0.335 9,mlp.experts.7.gate_proj,0.0004297771,0.01000,0.335 9,mlp.experts.8.gate_proj,0.0000641081,0.01000,0.336 9,mlp.experts.9.gate_proj,0.0001127264,0.01000,0.336 9,mlp.experts.10.gate_proj,0.0001411799,0.01000,0.337 9,mlp.experts.11.gate_proj,0.0000827012,0.01000,0.337 9,mlp.experts.12.gate_proj,0.0002201065,0.01000,0.339 9,mlp.experts.13.gate_proj,0.0000831598,0.01000,0.336 9,mlp.experts.14.gate_proj,0.0000474059,0.01000,0.337 9,mlp.experts.15.gate_proj,0.0000352080,0.01000,0.337 9,mlp.experts.16.gate_proj,0.0007102186,0.01000,0.336 9,mlp.experts.17.gate_proj,0.0002689828,0.01000,0.337 9,mlp.experts.18.gate_proj,0.0000394061,0.01000,0.335 9,mlp.experts.19.gate_proj,0.0002152947,0.01000,0.336 9,mlp.experts.20.gate_proj,0.0008147290,0.01000,0.337 9,mlp.experts.21.gate_proj,0.0037360759,0.01000,0.335 9,mlp.experts.22.gate_proj,0.0000319600,0.01000,0.336 9,mlp.experts.23.gate_proj,0.0002511529,0.01000,0.336 9,mlp.experts.24.gate_proj,0.0021306643,0.01000,0.337 9,mlp.experts.25.gate_proj,0.0000089340,0.01000,0.336 9,mlp.experts.26.gate_proj,0.0000658137,0.01000,0.336 9,mlp.experts.27.gate_proj,0.0000448244,0.01000,0.339 9,mlp.experts.28.gate_proj,0.0014211349,0.01000,0.339 9,mlp.experts.29.gate_proj,0.0001077983,0.01000,0.337 9,mlp.experts.30.gate_proj,0.0000179668,0.01000,0.337 9,mlp.experts.31.gate_proj,0.0000328113,0.01000,0.337 9,mlp.experts.32.gate_proj,0.0002971478,0.01000,0.336 9,mlp.experts.33.gate_proj,0.0003971087,0.01000,0.338 9,mlp.experts.34.gate_proj,0.0004380968,0.01000,0.335 9,mlp.experts.35.gate_proj,0.0006641191,0.01000,0.337 9,mlp.experts.36.gate_proj,0.0000486365,0.01000,0.336 9,mlp.experts.37.gate_proj,0.0001267609,0.01000,0.336 9,mlp.experts.38.gate_proj,0.0000750414,0.01000,0.334 9,mlp.experts.39.gate_proj,0.0001040190,0.01000,0.335 9,mlp.experts.40.gate_proj,0.0005171483,0.01000,0.338 9,mlp.experts.41.gate_proj,0.0000649565,0.01000,0.338 9,mlp.experts.42.gate_proj,0.0000194193,0.01000,0.336 9,mlp.experts.43.gate_proj,0.0001794520,0.01000,0.337 9,mlp.experts.44.gate_proj,0.0001201955,0.01000,0.337 9,mlp.experts.45.gate_proj,0.0001536762,0.01000,0.348 9,mlp.experts.46.gate_proj,0.0003639979,0.01000,0.336 9,mlp.experts.47.gate_proj,0.0003619569,0.01000,0.336 9,mlp.experts.48.gate_proj,0.0001405579,0.01000,0.335 9,mlp.experts.49.gate_proj,0.0000403433,0.01000,0.337 9,mlp.experts.50.gate_proj,0.0000709474,0.01000,0.336 9,mlp.experts.51.gate_proj,0.0020717506,0.01000,0.335 9,mlp.experts.52.gate_proj,0.0000528975,0.01000,0.335 9,mlp.experts.53.gate_proj,0.0004198682,0.01000,0.335 9,mlp.experts.54.gate_proj,0.0000289415,0.01000,0.338 9,mlp.experts.55.gate_proj,0.0000897649,0.01000,0.336 9,mlp.experts.56.gate_proj,0.0001210769,0.01000,0.338 9,mlp.experts.57.gate_proj,0.0003579357,0.01000,0.336 9,mlp.experts.58.gate_proj,0.0001728707,0.01000,0.335 9,mlp.experts.59.gate_proj,0.0004275595,0.01000,0.340 9,mlp.experts.60.gate_proj,0.0007718638,0.01000,0.335 9,mlp.experts.61.gate_proj,0.0001780296,0.01000,0.334 9,mlp.experts.62.gate_proj,0.0035266231,0.01000,0.335 9,mlp.experts.63.gate_proj,0.0000349795,0.01000,0.337 9,mlp.experts.0.up_proj,0.0000571366,0.01000,0.344 9,mlp.experts.1.up_proj,0.0009463808,0.01000,0.337 9,mlp.experts.2.up_proj,0.0001221378,0.01000,0.338 9,mlp.experts.3.up_proj,0.0000700853,0.01000,0.335 9,mlp.experts.4.up_proj,0.0000202587,0.01000,0.338 9,mlp.experts.5.up_proj,0.0001959410,0.01000,0.337 9,mlp.experts.6.up_proj,0.0000386023,0.01000,0.335 9,mlp.experts.7.up_proj,0.0003343723,0.01000,0.335 9,mlp.experts.8.up_proj,0.0000601676,0.01000,0.337 9,mlp.experts.9.up_proj,0.0000881346,0.01000,0.334 9,mlp.experts.10.up_proj,0.0001192741,0.01000,0.335 9,mlp.experts.11.up_proj,0.0000640652,0.01000,0.334 9,mlp.experts.12.up_proj,0.0001779666,0.01000,0.334 9,mlp.experts.13.up_proj,0.0000797743,0.01000,0.336 9,mlp.experts.14.up_proj,0.0000459414,0.01000,0.336 9,mlp.experts.15.up_proj,0.0000330677,0.01000,0.337 9,mlp.experts.16.up_proj,0.0006857616,0.01000,0.337 9,mlp.experts.17.up_proj,0.0001841429,0.01000,0.336 9,mlp.experts.18.up_proj,0.0000359697,0.01000,0.338 9,mlp.experts.19.up_proj,0.0001914949,0.01000,0.336 9,mlp.experts.20.up_proj,0.0006711770,0.01000,0.338 9,mlp.experts.21.up_proj,0.0029153045,0.01000,0.337 9,mlp.experts.22.up_proj,0.0000299837,0.01000,0.335 9,mlp.experts.23.up_proj,0.0002064574,0.01000,0.337 9,mlp.experts.24.up_proj,0.0017568442,0.01000,0.338 9,mlp.experts.25.up_proj,0.0000077351,0.01000,0.341 9,mlp.experts.26.up_proj,0.0000625869,0.01000,0.337 9,mlp.experts.27.up_proj,0.0000400368,0.01000,0.338 9,mlp.experts.28.up_proj,0.0010866499,0.01000,0.337 9,mlp.experts.29.up_proj,0.0000962703,0.01000,0.336 9,mlp.experts.30.up_proj,0.0000167238,0.01000,0.337 9,mlp.experts.31.up_proj,0.0000299293,0.01000,0.338 9,mlp.experts.32.up_proj,0.0002498312,0.01000,0.338 9,mlp.experts.33.up_proj,0.0003634101,0.01000,0.338 9,mlp.experts.34.up_proj,0.0003297841,0.01000,0.336 9,mlp.experts.35.up_proj,0.0005898482,0.01000,0.335 9,mlp.experts.36.up_proj,0.0000456033,0.01000,0.336 9,mlp.experts.37.up_proj,0.0000969710,0.01000,0.336 9,mlp.experts.38.up_proj,0.0000652413,0.01000,0.336 9,mlp.experts.39.up_proj,0.0000989967,0.01000,0.335 9,mlp.experts.40.up_proj,0.0004351831,0.01000,0.339 9,mlp.experts.41.up_proj,0.0000553553,0.01000,0.340 9,mlp.experts.42.up_proj,0.0000156348,0.01000,0.337 9,mlp.experts.43.up_proj,0.0001599782,0.01000,0.337 9,mlp.experts.44.up_proj,0.0000963162,0.01000,0.339 9,mlp.experts.45.up_proj,0.0001431470,0.01000,0.338 9,mlp.experts.46.up_proj,0.0003300538,0.01000,0.335 9,mlp.experts.47.up_proj,0.0002551818,0.01000,0.339 9,mlp.experts.48.up_proj,0.0001057153,0.01000,0.339 9,mlp.experts.49.up_proj,0.0000357791,0.01000,0.338 9,mlp.experts.50.up_proj,0.0000679343,0.01000,0.335 9,mlp.experts.51.up_proj,0.0015458640,0.01000,0.339 9,mlp.experts.52.up_proj,0.0000462032,0.01000,0.335 9,mlp.experts.53.up_proj,0.0003641352,0.01000,0.335 9,mlp.experts.54.up_proj,0.0000272458,0.01000,0.338 9,mlp.experts.55.up_proj,0.0000724859,0.01000,0.336 9,mlp.experts.56.up_proj,0.0001112294,0.01000,0.334 9,mlp.experts.57.up_proj,0.0002857551,0.01000,0.336 9,mlp.experts.58.up_proj,0.0001584910,0.01000,0.336 9,mlp.experts.59.up_proj,0.0003396685,0.01000,0.336 9,mlp.experts.60.up_proj,0.0006727652,0.01000,0.338 9,mlp.experts.61.up_proj,0.0001672828,0.01000,0.337 9,mlp.experts.62.up_proj,0.0027056307,0.01000,0.335 9,mlp.experts.63.up_proj,0.0000309421,0.01000,0.334 9,mlp.experts.0.down_proj,0.0000046936,0.01000,0.240 9,mlp.experts.1.down_proj,0.0000916991,0.01000,0.231 9,mlp.experts.2.down_proj,0.0000074292,0.01000,0.231 9,mlp.experts.3.down_proj,0.0000066908,0.01000,0.233 9,mlp.experts.4.down_proj,0.0000016622,0.01000,0.232 9,mlp.experts.5.down_proj,0.0000125292,0.01000,0.231 9,mlp.experts.6.down_proj,0.0000032840,0.01000,0.231 9,mlp.experts.7.down_proj,0.0000328297,0.01000,0.248 9,mlp.experts.8.down_proj,0.0000048177,0.01000,0.239 9,mlp.experts.9.down_proj,0.0000070201,0.01000,0.232 9,mlp.experts.10.down_proj,0.0000102772,0.01000,0.231 9,mlp.experts.11.down_proj,0.0000058921,0.01000,0.234 9,mlp.experts.12.down_proj,0.0000166922,0.01000,0.231 9,mlp.experts.13.down_proj,0.0000053951,0.01000,0.231 9,mlp.experts.14.down_proj,0.0000038591,0.01000,0.231 9,mlp.experts.15.down_proj,0.0000024913,0.01000,0.230 9,mlp.experts.16.down_proj,0.0000535279,0.01000,0.232 9,mlp.experts.17.down_proj,0.0000176449,0.01000,0.232 9,mlp.experts.18.down_proj,0.0000021492,0.01000,0.231 9,mlp.experts.19.down_proj,0.0000137592,0.01000,0.231 9,mlp.experts.20.down_proj,0.0000507772,0.01000,0.230 9,mlp.experts.21.down_proj,0.0002467156,0.01000,0.231 9,mlp.experts.22.down_proj,0.0000018167,0.01000,0.229 9,mlp.experts.23.down_proj,0.0000165812,0.01000,0.230 9,mlp.experts.24.down_proj,0.0001252499,0.01000,0.230 9,mlp.experts.25.down_proj,0.0000004741,0.01000,0.232 9,mlp.experts.26.down_proj,0.0000052590,0.01000,0.232 9,mlp.experts.27.down_proj,0.0000030045,0.01000,0.228 9,mlp.experts.28.down_proj,0.0001027534,0.01000,0.231 9,mlp.experts.29.down_proj,0.0000058521,0.01000,0.230 9,mlp.experts.30.down_proj,0.0000011991,0.01000,0.231 9,mlp.experts.31.down_proj,0.0000016135,0.01000,0.231 9,mlp.experts.32.down_proj,0.0000186589,0.01000,0.229 9,mlp.experts.33.down_proj,0.0000370521,0.01000,0.230 9,mlp.experts.34.down_proj,0.0000282169,0.01000,0.228 9,mlp.experts.35.down_proj,0.0000423478,0.01000,0.229 9,mlp.experts.36.down_proj,0.0000033812,0.01000,0.229 9,mlp.experts.37.down_proj,0.0000074286,0.01000,0.229 9,mlp.experts.38.down_proj,0.0000057433,0.01000,0.230 9,mlp.experts.39.down_proj,0.0000087048,0.01000,0.230 9,mlp.experts.40.down_proj,0.0000395352,0.01000,0.229 9,mlp.experts.41.down_proj,0.0000043212,0.01000,0.231 9,mlp.experts.42.down_proj,0.0000009053,0.01000,0.230 9,mlp.experts.43.down_proj,0.0000118091,0.01000,0.231 9,mlp.experts.44.down_proj,0.0000089237,0.01000,0.230 9,mlp.experts.45.down_proj,0.0000095931,0.01000,0.231 9,mlp.experts.46.down_proj,0.0000207223,0.01000,0.229 9,mlp.experts.47.down_proj,0.0000231571,0.01000,0.229 9,mlp.experts.48.down_proj,0.0000102720,0.01000,0.230 9,mlp.experts.49.down_proj,0.0000024301,0.01000,0.230 9,mlp.experts.50.down_proj,0.0000053296,0.01000,0.231 9,mlp.experts.51.down_proj,0.0001316853,0.01000,0.232 9,mlp.experts.52.down_proj,0.0000037967,0.01000,0.230 9,mlp.experts.53.down_proj,0.0000285642,0.01000,0.230 9,mlp.experts.54.down_proj,0.0000020857,0.01000,0.230 9,mlp.experts.55.down_proj,0.0000057305,0.01000,0.230 9,mlp.experts.56.down_proj,0.0000092989,0.01000,0.230 9,mlp.experts.57.down_proj,0.0000222063,0.01000,0.229 9,mlp.experts.58.down_proj,0.0000099844,0.01000,0.229 9,mlp.experts.59.down_proj,0.0000220208,0.01000,0.228 9,mlp.experts.60.down_proj,0.0000574177,0.01000,0.229 9,mlp.experts.61.down_proj,0.0000100839,0.01000,0.229 9,mlp.experts.62.down_proj,0.0002307543,0.01000,0.229 9,mlp.experts.63.down_proj,0.0000018073,0.01000,0.230 9,mlp.shared_experts.gate_proj,0.0000089074,0.01000,0.342 9,mlp.shared_experts.up_proj,0.0000085315,0.01000,0.335 9,mlp.shared_experts.down_proj,0.0000003722,0.01000,0.470 10,self_attn.kv_a_proj_with_mqa,0.0000061701,0.01000,0.335 10,self_attn.kv_b_proj,0.0000915958,0.01000,0.094 10,self_attn.o_proj,0.0000010785,0.01000,0.341 10,mlp.experts.0.gate_proj,0.0001031203,0.01000,0.346 10,mlp.experts.1.gate_proj,0.0001095399,0.01000,0.335 10,mlp.experts.2.gate_proj,0.0008710645,0.01000,0.336 10,mlp.experts.3.gate_proj,0.0000754377,0.01000,0.338 10,mlp.experts.4.gate_proj,0.0000145616,0.01000,0.336 10,mlp.experts.5.gate_proj,0.0002241261,0.01000,0.334 10,mlp.experts.6.gate_proj,0.0005526877,0.01000,0.336 10,mlp.experts.7.gate_proj,0.0000866720,0.01000,0.334 10,mlp.experts.8.gate_proj,0.0021034770,0.01000,0.335 10,mlp.experts.9.gate_proj,0.0002283632,0.01000,0.337 10,mlp.experts.10.gate_proj,0.0000228554,0.01000,0.334 10,mlp.experts.11.gate_proj,0.0028673457,0.01000,0.335 10,mlp.experts.12.gate_proj,0.0000351183,0.01000,0.337 10,mlp.experts.13.gate_proj,0.0001581966,0.01000,0.336 10,mlp.experts.14.gate_proj,0.0004344151,0.01000,0.335 10,mlp.experts.15.gate_proj,0.0012417549,0.01000,0.335 10,mlp.experts.16.gate_proj,0.0031430262,0.01000,0.336 10,mlp.experts.17.gate_proj,0.0001430411,0.01000,0.334 10,mlp.experts.18.gate_proj,0.0000457625,0.01000,0.337 10,mlp.experts.19.gate_proj,0.0001400857,0.01000,0.337 10,mlp.experts.20.gate_proj,0.0005721138,0.01000,0.334 10,mlp.experts.21.gate_proj,0.0005051109,0.01000,0.337 10,mlp.experts.22.gate_proj,0.0004246828,0.01000,0.336 10,mlp.experts.23.gate_proj,0.0000421485,0.01000,0.335 10,mlp.experts.24.gate_proj,0.0035126141,0.01000,0.337 10,mlp.experts.25.gate_proj,0.0006712115,0.01000,0.336 10,mlp.experts.26.gate_proj,0.0006048336,0.01000,0.335 10,mlp.experts.27.gate_proj,0.0001397221,0.01000,0.335 10,mlp.experts.28.gate_proj,0.0002577486,0.01000,0.337 10,mlp.experts.29.gate_proj,0.0001144638,0.01000,0.335 10,mlp.experts.30.gate_proj,0.0000158504,0.01000,0.337 10,mlp.experts.31.gate_proj,0.0000494232,0.01000,0.337 10,mlp.experts.32.gate_proj,0.0035699245,0.01000,0.337 10,mlp.experts.33.gate_proj,0.0010556318,0.01000,0.346 10,mlp.experts.34.gate_proj,0.0004664926,0.01000,0.336 10,mlp.experts.35.gate_proj,0.0023486875,0.01000,0.337 10,mlp.experts.36.gate_proj,0.0000659611,0.01000,0.334 10,mlp.experts.37.gate_proj,0.0025418201,0.01000,0.338 10,mlp.experts.38.gate_proj,0.0000531942,0.01000,0.337 10,mlp.experts.39.gate_proj,0.0000940828,0.01000,0.335 10,mlp.experts.40.gate_proj,0.0001706032,0.01000,0.336 10,mlp.experts.41.gate_proj,0.0000646051,0.01000,0.338 10,mlp.experts.42.gate_proj,0.0011320717,0.01000,0.337 10,mlp.experts.43.gate_proj,0.0006772464,0.01000,0.335 10,mlp.experts.44.gate_proj,0.0006312488,0.01000,0.338 10,mlp.experts.45.gate_proj,0.0000261397,0.01000,0.336 10,mlp.experts.46.gate_proj,0.0004738357,0.01000,0.336 10,mlp.experts.47.gate_proj,0.0000515850,0.01000,0.341 10,mlp.experts.48.gate_proj,0.0025748539,0.01000,0.340 10,mlp.experts.49.gate_proj,0.0000193299,0.01000,0.343 10,mlp.experts.50.gate_proj,0.0000607917,0.01000,0.337 10,mlp.experts.51.gate_proj,0.0000220150,0.01000,0.339 10,mlp.experts.52.gate_proj,0.0000207443,0.01000,0.338 10,mlp.experts.53.gate_proj,0.0000253440,0.01000,0.338 10,mlp.experts.54.gate_proj,0.0000253518,0.01000,0.340 10,mlp.experts.55.gate_proj,0.0000366916,0.01000,0.341 10,mlp.experts.56.gate_proj,0.0030145665,0.01000,0.336 10,mlp.experts.57.gate_proj,0.0001617754,0.01000,0.338 10,mlp.experts.58.gate_proj,0.0000270967,0.01000,0.339 10,mlp.experts.59.gate_proj,0.0000190560,0.01000,0.337 10,mlp.experts.60.gate_proj,0.0001987902,0.01000,0.336 10,mlp.experts.61.gate_proj,0.0009916597,0.01000,0.338 10,mlp.experts.62.gate_proj,0.0001176423,0.01000,0.339 10,mlp.experts.63.gate_proj,0.0003353402,0.01000,0.336 10,mlp.experts.0.up_proj,0.0000943389,0.01000,0.348 10,mlp.experts.1.up_proj,0.0001021575,0.01000,0.337 10,mlp.experts.2.up_proj,0.0007096347,0.01000,0.337 10,mlp.experts.3.up_proj,0.0000718602,0.01000,0.338 10,mlp.experts.4.up_proj,0.0000121556,0.01000,0.339 10,mlp.experts.5.up_proj,0.0001869092,0.01000,0.337 10,mlp.experts.6.up_proj,0.0004793978,0.01000,0.335 10,mlp.experts.7.up_proj,0.0000808105,0.01000,0.338 10,mlp.experts.8.up_proj,0.0017146554,0.01000,0.336 10,mlp.experts.9.up_proj,0.0002044144,0.01000,0.336 10,mlp.experts.10.up_proj,0.0000210582,0.01000,0.339 10,mlp.experts.11.up_proj,0.0026466949,0.01000,0.337 10,mlp.experts.12.up_proj,0.0000303673,0.01000,0.337 10,mlp.experts.13.up_proj,0.0001417579,0.01000,0.338 10,mlp.experts.14.up_proj,0.0003683459,0.01000,0.337 10,mlp.experts.15.up_proj,0.0010339016,0.01000,0.339 10,mlp.experts.16.up_proj,0.0028999112,0.01000,0.338 10,mlp.experts.17.up_proj,0.0001200705,0.01000,0.339 10,mlp.experts.18.up_proj,0.0000430817,0.01000,0.338 10,mlp.experts.19.up_proj,0.0001209696,0.01000,0.336 10,mlp.experts.20.up_proj,0.0004756403,0.01000,0.338 10,mlp.experts.21.up_proj,0.0004079450,0.01000,0.337 10,mlp.experts.22.up_proj,0.0003733458,0.01000,0.336 10,mlp.experts.23.up_proj,0.0000389925,0.01000,0.336 10,mlp.experts.24.up_proj,0.0027421345,0.01000,0.338 10,mlp.experts.25.up_proj,0.0005783423,0.01000,0.336 10,mlp.experts.26.up_proj,0.0004749819,0.01000,0.336 10,mlp.experts.27.up_proj,0.0000997363,0.01000,0.337 10,mlp.experts.28.up_proj,0.0002540331,0.01000,0.339 10,mlp.experts.29.up_proj,0.0001093126,0.01000,0.336 10,mlp.experts.30.up_proj,0.0000138972,0.01000,0.338 10,mlp.experts.31.up_proj,0.0000461668,0.01000,0.340 10,mlp.experts.32.up_proj,0.0029496001,0.01000,0.337 10,mlp.experts.33.up_proj,0.0009541618,0.01000,0.338 10,mlp.experts.34.up_proj,0.0003535570,0.01000,0.336 10,mlp.experts.35.up_proj,0.0018408395,0.01000,0.336 10,mlp.experts.36.up_proj,0.0000607144,0.01000,0.340 10,mlp.experts.37.up_proj,0.0021987460,0.01000,0.335 10,mlp.experts.38.up_proj,0.0000441121,0.01000,0.337 10,mlp.experts.39.up_proj,0.0000876389,0.01000,0.334 10,mlp.experts.40.up_proj,0.0001242425,0.01000,0.334 10,mlp.experts.41.up_proj,0.0000601634,0.01000,0.334 10,mlp.experts.42.up_proj,0.0009333436,0.01000,0.336 10,mlp.experts.43.up_proj,0.0006027189,0.01000,0.335 10,mlp.experts.44.up_proj,0.0004450529,0.01000,0.336 10,mlp.experts.45.up_proj,0.0000223317,0.01000,0.336 10,mlp.experts.46.up_proj,0.0004195724,0.01000,0.337 10,mlp.experts.47.up_proj,0.0000484965,0.01000,0.337 10,mlp.experts.48.up_proj,0.0020627387,0.01000,0.336 10,mlp.experts.49.up_proj,0.0000157685,0.01000,0.336 10,mlp.experts.50.up_proj,0.0000581185,0.01000,0.336 10,mlp.experts.51.up_proj,0.0000208455,0.01000,0.335 10,mlp.experts.52.up_proj,0.0000192235,0.01000,0.336 10,mlp.experts.53.up_proj,0.0000232632,0.01000,0.337 10,mlp.experts.54.up_proj,0.0000214475,0.01000,0.335 10,mlp.experts.55.up_proj,0.0000298750,0.01000,0.336 10,mlp.experts.56.up_proj,0.0025332101,0.01000,0.333 10,mlp.experts.57.up_proj,0.0001463683,0.01000,0.335 10,mlp.experts.58.up_proj,0.0000247664,0.01000,0.334 10,mlp.experts.59.up_proj,0.0000180248,0.01000,0.335 10,mlp.experts.60.up_proj,0.0001415151,0.01000,0.340 10,mlp.experts.61.up_proj,0.0010042263,0.01000,0.334 10,mlp.experts.62.up_proj,0.0001044898,0.01000,0.337 10,mlp.experts.63.up_proj,0.0002959106,0.01000,0.336 10,mlp.experts.0.down_proj,0.0000087387,0.01000,0.240 10,mlp.experts.1.down_proj,0.0000081337,0.01000,0.230 10,mlp.experts.2.down_proj,0.0000707215,0.01000,0.231 10,mlp.experts.3.down_proj,0.0000057578,0.01000,0.229 10,mlp.experts.4.down_proj,0.0000007819,0.01000,0.230 10,mlp.experts.5.down_proj,0.0000167182,0.01000,0.229 10,mlp.experts.6.down_proj,0.0000365502,0.01000,0.229 10,mlp.experts.7.down_proj,0.0000050379,0.01000,0.229 10,mlp.experts.8.down_proj,0.0001586727,0.01000,0.231 10,mlp.experts.9.down_proj,0.0000184872,0.01000,0.230 10,mlp.experts.10.down_proj,0.0000015852,0.01000,0.230 10,mlp.experts.11.down_proj,0.0002078165,0.01000,0.230 10,mlp.experts.12.down_proj,0.0000021664,0.01000,0.230 10,mlp.experts.13.down_proj,0.0000087615,0.01000,0.230 10,mlp.experts.14.down_proj,0.0000341574,0.01000,0.230 10,mlp.experts.15.down_proj,0.0000937861,0.01000,0.230 10,mlp.experts.16.down_proj,0.0002282882,0.01000,0.229 10,mlp.experts.17.down_proj,0.0000101126,0.01000,0.234 10,mlp.experts.18.down_proj,0.0000029370,0.01000,0.231 10,mlp.experts.19.down_proj,0.0000079705,0.01000,0.231 10,mlp.experts.20.down_proj,0.0000464315,0.01000,0.232 10,mlp.experts.21.down_proj,0.0000386742,0.01000,0.230 10,mlp.experts.22.down_proj,0.0000290859,0.01000,0.232 10,mlp.experts.23.down_proj,0.0000031470,0.01000,0.233 10,mlp.experts.24.down_proj,0.0002870018,0.01000,0.231 10,mlp.experts.25.down_proj,0.0000511106,0.01000,0.231 10,mlp.experts.26.down_proj,0.0000410467,0.01000,0.230 10,mlp.experts.27.down_proj,0.0000099062,0.01000,0.231 10,mlp.experts.28.down_proj,0.0000193555,0.01000,0.230 10,mlp.experts.29.down_proj,0.0000089996,0.01000,0.231 10,mlp.experts.30.down_proj,0.0000010261,0.01000,0.241 10,mlp.experts.31.down_proj,0.0000037066,0.01000,0.232 10,mlp.experts.32.down_proj,0.0003241796,0.01000,0.232 10,mlp.experts.33.down_proj,0.0000630977,0.01000,0.231 10,mlp.experts.34.down_proj,0.0000281878,0.01000,0.231 10,mlp.experts.35.down_proj,0.0001710672,0.01000,0.229 10,mlp.experts.36.down_proj,0.0000038573,0.01000,0.232 10,mlp.experts.37.down_proj,0.0001587958,0.01000,0.230 10,mlp.experts.38.down_proj,0.0000024318,0.01000,0.231 10,mlp.experts.39.down_proj,0.0000055137,0.01000,0.232 10,mlp.experts.40.down_proj,0.0000111860,0.01000,0.231 10,mlp.experts.41.down_proj,0.0000044414,0.01000,0.231 10,mlp.experts.42.down_proj,0.0000940722,0.01000,0.230 10,mlp.experts.43.down_proj,0.0000406955,0.01000,0.232 10,mlp.experts.44.down_proj,0.0000503462,0.01000,0.230 10,mlp.experts.45.down_proj,0.0000012336,0.01000,0.233 10,mlp.experts.46.down_proj,0.0000249151,0.01000,0.231 10,mlp.experts.47.down_proj,0.0000032401,0.01000,0.235 10,mlp.experts.48.down_proj,0.0001276508,0.01000,0.232 10,mlp.experts.49.down_proj,0.0000014983,0.01000,0.231 10,mlp.experts.50.down_proj,0.0000054629,0.01000,0.231 10,mlp.experts.51.down_proj,0.0000016051,0.01000,0.231 10,mlp.experts.52.down_proj,0.0000014367,0.01000,0.232 10,mlp.experts.53.down_proj,0.0000019131,0.01000,0.232 10,mlp.experts.54.down_proj,0.0000012845,0.01000,0.232 10,mlp.experts.55.down_proj,0.0000020479,0.01000,0.232 10,mlp.experts.56.down_proj,0.0002272974,0.01000,0.232 10,mlp.experts.57.down_proj,0.0000110673,0.01000,0.230 10,mlp.experts.58.down_proj,0.0000021481,0.01000,0.231 10,mlp.experts.59.down_proj,0.0000014119,0.01000,0.230 10,mlp.experts.60.down_proj,0.0000134970,0.01000,0.232 10,mlp.experts.61.down_proj,0.0000759288,0.01000,0.230 10,mlp.experts.62.down_proj,0.0000093427,0.01000,0.232 10,mlp.experts.63.down_proj,0.0000276714,0.01000,0.232 10,mlp.shared_experts.gate_proj,0.0000087672,0.01000,0.343 10,mlp.shared_experts.up_proj,0.0000084208,0.01000,0.332 10,mlp.shared_experts.down_proj,0.0000004641,0.01000,0.467 11,self_attn.kv_a_proj_with_mqa,0.0000070654,0.01000,0.340 11,self_attn.kv_b_proj,0.0001066084,0.01000,0.095 11,self_attn.o_proj,0.0000018257,0.01000,0.344 11,mlp.experts.0.gate_proj,0.0001952327,0.01000,0.347 11,mlp.experts.1.gate_proj,0.0009035554,0.01000,0.336 11,mlp.experts.2.gate_proj,0.0017238098,0.01000,0.336 11,mlp.experts.3.gate_proj,0.0000204549,0.01000,0.336 11,mlp.experts.4.gate_proj,0.0000446515,0.01000,0.337 11,mlp.experts.5.gate_proj,0.0000760582,0.01000,0.338 11,mlp.experts.6.gate_proj,0.0001276290,0.01000,0.338 11,mlp.experts.7.gate_proj,0.0001027024,0.01000,0.336 11,mlp.experts.8.gate_proj,0.0002306158,0.01000,0.342 11,mlp.experts.9.gate_proj,0.0000163527,0.01000,0.340 11,mlp.experts.10.gate_proj,0.0002198992,0.01000,0.335 11,mlp.experts.11.gate_proj,0.0000792976,0.01000,0.337 11,mlp.experts.12.gate_proj,0.0001488736,0.01000,0.338 11,mlp.experts.13.gate_proj,0.0067425337,0.01000,0.336 11,mlp.experts.14.gate_proj,0.0000327798,0.01000,0.336 11,mlp.experts.15.gate_proj,0.0031235964,0.01000,0.337 11,mlp.experts.16.gate_proj,0.0000482239,0.01000,0.336 11,mlp.experts.17.gate_proj,0.0000461367,0.01000,0.336 11,mlp.experts.18.gate_proj,0.0011348270,0.01000,0.338 11,mlp.experts.19.gate_proj,0.0000783517,0.01000,0.337 11,mlp.experts.20.gate_proj,0.0000640038,0.01000,0.336 11,mlp.experts.21.gate_proj,0.0002637908,0.01000,0.336 11,mlp.experts.22.gate_proj,0.0000381554,0.01000,0.338 11,mlp.experts.23.gate_proj,0.0001922597,0.01000,0.338 11,mlp.experts.24.gate_proj,0.0000372172,0.01000,0.336 11,mlp.experts.25.gate_proj,0.0000191580,0.01000,0.337 11,mlp.experts.26.gate_proj,0.0025878604,0.01000,0.336 11,mlp.experts.27.gate_proj,0.0057602725,0.01000,0.336 11,mlp.experts.28.gate_proj,0.0025972978,0.01000,0.338 11,mlp.experts.29.gate_proj,0.0014862844,0.01000,0.336 11,mlp.experts.30.gate_proj,0.0002601003,0.01000,0.337 11,mlp.experts.31.gate_proj,0.0000473429,0.01000,0.338 11,mlp.experts.32.gate_proj,0.0003475963,0.01000,0.339 11,mlp.experts.33.gate_proj,0.0017029153,0.01000,0.336 11,mlp.experts.34.gate_proj,0.0009762675,0.01000,0.338 11,mlp.experts.35.gate_proj,0.0000897772,0.01000,0.337 11,mlp.experts.36.gate_proj,0.0008885863,0.01000,0.336 11,mlp.experts.37.gate_proj,0.0000477515,0.01000,0.337 11,mlp.experts.38.gate_proj,0.0001535856,0.01000,0.340 11,mlp.experts.39.gate_proj,0.0008007819,0.01000,0.340 11,mlp.experts.40.gate_proj,0.0002222539,0.01000,0.339 11,mlp.experts.41.gate_proj,0.0000255581,0.01000,0.337 11,mlp.experts.42.gate_proj,0.0006192183,0.01000,0.336 11,mlp.experts.43.gate_proj,0.0000965532,0.01000,0.336 11,mlp.experts.44.gate_proj,0.0000239495,0.01000,0.336 11,mlp.experts.45.gate_proj,0.0001312237,0.01000,0.338 11,mlp.experts.46.gate_proj,0.0008504588,0.01000,0.338 11,mlp.experts.47.gate_proj,0.0000759314,0.01000,0.335 11,mlp.experts.48.gate_proj,0.0000995396,0.01000,0.337 11,mlp.experts.49.gate_proj,0.0000688622,0.01000,0.340 11,mlp.experts.50.gate_proj,0.0008885849,0.01000,0.337 11,mlp.experts.51.gate_proj,0.0000310889,0.01000,0.336 11,mlp.experts.52.gate_proj,0.0003525239,0.01000,0.338 11,mlp.experts.53.gate_proj,0.0014431857,0.01000,0.336 11,mlp.experts.54.gate_proj,0.0000501241,0.01000,0.336 11,mlp.experts.55.gate_proj,0.0010957819,0.01000,0.335 11,mlp.experts.56.gate_proj,0.0000550727,0.01000,0.335 11,mlp.experts.57.gate_proj,0.0027143652,0.01000,0.337 11,mlp.experts.58.gate_proj,0.0000179654,0.01000,0.336 11,mlp.experts.59.gate_proj,0.0000909822,0.01000,0.337 11,mlp.experts.60.gate_proj,0.0010016511,0.01000,0.337 11,mlp.experts.61.gate_proj,0.0000801097,0.01000,0.336 11,mlp.experts.62.gate_proj,0.0000123646,0.01000,0.333 11,mlp.experts.63.gate_proj,0.0006699553,0.01000,0.332 11,mlp.experts.0.up_proj,0.0001621751,0.01000,0.345 11,mlp.experts.1.up_proj,0.0007453765,0.01000,0.339 11,mlp.experts.2.up_proj,0.0012170819,0.01000,0.336 11,mlp.experts.3.up_proj,0.0000191163,0.01000,0.339 11,mlp.experts.4.up_proj,0.0000421599,0.01000,0.339 11,mlp.experts.5.up_proj,0.0000715247,0.01000,0.335 11,mlp.experts.6.up_proj,0.0001099364,0.01000,0.336 11,mlp.experts.7.up_proj,0.0000920617,0.01000,0.338 11,mlp.experts.8.up_proj,0.0002051365,0.01000,0.339 11,mlp.experts.9.up_proj,0.0000133379,0.01000,0.337 11,mlp.experts.10.up_proj,0.0001994345,0.01000,0.340 11,mlp.experts.11.up_proj,0.0000721949,0.01000,0.339 11,mlp.experts.12.up_proj,0.0001312071,0.01000,0.336 11,mlp.experts.13.up_proj,0.0058976884,0.01000,0.337 11,mlp.experts.14.up_proj,0.0000304419,0.01000,0.338 11,mlp.experts.15.up_proj,0.0030652375,0.01000,0.339 11,mlp.experts.16.up_proj,0.0000418792,0.01000,0.337 11,mlp.experts.17.up_proj,0.0000433045,0.01000,0.338 11,mlp.experts.18.up_proj,0.0010250964,0.01000,0.339 11,mlp.experts.19.up_proj,0.0000698879,0.01000,0.336 11,mlp.experts.20.up_proj,0.0000475726,0.01000,0.334 11,mlp.experts.21.up_proj,0.0002526921,0.01000,0.335 11,mlp.experts.22.up_proj,0.0000353034,0.01000,0.340 11,mlp.experts.23.up_proj,0.0001690837,0.01000,0.338 11,mlp.experts.24.up_proj,0.0000313770,0.01000,0.341 11,mlp.experts.25.up_proj,0.0000179115,0.01000,0.338 11,mlp.experts.26.up_proj,0.0025053148,0.01000,0.338 11,mlp.experts.27.up_proj,0.0046194315,0.01000,0.339 11,mlp.experts.28.up_proj,0.0020127687,0.01000,0.336 11,mlp.experts.29.up_proj,0.0012989922,0.01000,0.335 11,mlp.experts.30.up_proj,0.0002444649,0.01000,0.337 11,mlp.experts.31.up_proj,0.0000438509,0.01000,0.335 11,mlp.experts.32.up_proj,0.0002912838,0.01000,0.336 11,mlp.experts.33.up_proj,0.0014147821,0.01000,0.336 11,mlp.experts.34.up_proj,0.0007604525,0.01000,0.338 11,mlp.experts.35.up_proj,0.0000779438,0.01000,0.336 11,mlp.experts.36.up_proj,0.0007174594,0.01000,0.336 11,mlp.experts.37.up_proj,0.0000439528,0.01000,0.335 11,mlp.experts.38.up_proj,0.0001243640,0.01000,0.334 11,mlp.experts.39.up_proj,0.0007068163,0.01000,0.336 11,mlp.experts.40.up_proj,0.0001795330,0.01000,0.334 11,mlp.experts.41.up_proj,0.0000229449,0.01000,0.336 11,mlp.experts.42.up_proj,0.0005367233,0.01000,0.337 11,mlp.experts.43.up_proj,0.0000886196,0.01000,0.335 11,mlp.experts.44.up_proj,0.0000228060,0.01000,0.338 11,mlp.experts.45.up_proj,0.0001143003,0.01000,0.337 11,mlp.experts.46.up_proj,0.0007262975,0.01000,0.335 11,mlp.experts.47.up_proj,0.0000694263,0.01000,0.335 11,mlp.experts.48.up_proj,0.0000800902,0.01000,0.335 11,mlp.experts.49.up_proj,0.0000638725,0.01000,0.341 11,mlp.experts.50.up_proj,0.0007288112,0.01000,0.343 11,mlp.experts.51.up_proj,0.0000267283,0.01000,0.335 11,mlp.experts.52.up_proj,0.0002967348,0.01000,0.336 11,mlp.experts.53.up_proj,0.0010469749,0.01000,0.335 11,mlp.experts.54.up_proj,0.0000457114,0.01000,0.333 11,mlp.experts.55.up_proj,0.0009625306,0.01000,0.334 11,mlp.experts.56.up_proj,0.0000481929,0.01000,0.334 11,mlp.experts.57.up_proj,0.0025006692,0.01000,0.335 11,mlp.experts.58.up_proj,0.0000135434,0.01000,0.335 11,mlp.experts.59.up_proj,0.0000794198,0.01000,0.334 11,mlp.experts.60.up_proj,0.0009648779,0.01000,0.334 11,mlp.experts.61.up_proj,0.0000726932,0.01000,0.334 11,mlp.experts.62.up_proj,0.0000106961,0.01000,0.334 11,mlp.experts.63.up_proj,0.0005344354,0.01000,0.333 11,mlp.experts.0.down_proj,0.0000176119,0.01000,0.239 11,mlp.experts.1.down_proj,0.0000648175,0.01000,0.230 11,mlp.experts.2.down_proj,0.0001195465,0.01000,0.229 11,mlp.experts.3.down_proj,0.0000016599,0.01000,0.230 11,mlp.experts.4.down_proj,0.0000041601,0.01000,0.231 11,mlp.experts.5.down_proj,0.0000068484,0.01000,0.229 11,mlp.experts.6.down_proj,0.0000079591,0.01000,0.230 11,mlp.experts.7.down_proj,0.0000067734,0.01000,0.230 11,mlp.experts.8.down_proj,0.0000156601,0.01000,0.231 11,mlp.experts.9.down_proj,0.0000008676,0.01000,0.230 11,mlp.experts.10.down_proj,0.0000191637,0.01000,0.231 11,mlp.experts.11.down_proj,0.0000058182,0.01000,0.230 11,mlp.experts.12.down_proj,0.0000119807,0.01000,0.230 11,mlp.experts.13.down_proj,0.0004647648,0.01000,0.229 11,mlp.experts.14.down_proj,0.0000028004,0.01000,0.230 11,mlp.experts.15.down_proj,0.0002466777,0.01000,0.229 11,mlp.experts.16.down_proj,0.0000038639,0.01000,0.232 11,mlp.experts.17.down_proj,0.0000035187,0.01000,0.229 11,mlp.experts.18.down_proj,0.0000891968,0.01000,0.230 11,mlp.experts.19.down_proj,0.0000065139,0.01000,0.229 11,mlp.experts.20.down_proj,0.0000043986,0.01000,0.231 11,mlp.experts.21.down_proj,0.0000205790,0.01000,0.230 11,mlp.experts.22.down_proj,0.0000036695,0.01000,0.230 11,mlp.experts.23.down_proj,0.0000161472,0.01000,0.230 11,mlp.experts.24.down_proj,0.0000035801,0.01000,0.230 11,mlp.experts.25.down_proj,0.0000021305,0.01000,0.229 11,mlp.experts.26.down_proj,0.0002075584,0.01000,0.229 11,mlp.experts.27.down_proj,0.0004176449,0.01000,0.230 11,mlp.experts.28.down_proj,0.0002124291,0.01000,0.230 11,mlp.experts.29.down_proj,0.0001156352,0.01000,0.229 11,mlp.experts.30.down_proj,0.0000216302,0.01000,0.236 11,mlp.experts.31.down_proj,0.0000042372,0.01000,0.230 11,mlp.experts.32.down_proj,0.0000285168,0.01000,0.231 11,mlp.experts.33.down_proj,0.0001381340,0.01000,0.231 11,mlp.experts.34.down_proj,0.0000768989,0.01000,0.233 11,mlp.experts.35.down_proj,0.0000073425,0.01000,0.233 11,mlp.experts.36.down_proj,0.0000671656,0.01000,0.233 11,mlp.experts.37.down_proj,0.0000033963,0.01000,0.231 11,mlp.experts.38.down_proj,0.0000113346,0.01000,0.232 11,mlp.experts.39.down_proj,0.0000657061,0.01000,0.232 11,mlp.experts.40.down_proj,0.0000153659,0.01000,0.232 11,mlp.experts.41.down_proj,0.0000015551,0.01000,0.234 11,mlp.experts.42.down_proj,0.0000502557,0.01000,0.231 11,mlp.experts.43.down_proj,0.0000075011,0.01000,0.232 11,mlp.experts.44.down_proj,0.0000023231,0.01000,0.231 11,mlp.experts.45.down_proj,0.0000106667,0.01000,0.231 11,mlp.experts.46.down_proj,0.0000686189,0.01000,0.232 11,mlp.experts.47.down_proj,0.0000055376,0.01000,0.230 11,mlp.experts.48.down_proj,0.0000107216,0.01000,0.232 11,mlp.experts.49.down_proj,0.0000052325,0.01000,0.231 11,mlp.experts.50.down_proj,0.0000646964,0.01000,0.232 11,mlp.experts.51.down_proj,0.0000022619,0.01000,0.232 11,mlp.experts.52.down_proj,0.0000259727,0.01000,0.231 11,mlp.experts.53.down_proj,0.0001062157,0.01000,0.232 11,mlp.experts.54.down_proj,0.0000041385,0.01000,0.234 11,mlp.experts.55.down_proj,0.0000823076,0.01000,0.231 11,mlp.experts.56.down_proj,0.0000053053,0.01000,0.232 11,mlp.experts.57.down_proj,0.0001894476,0.01000,0.232 11,mlp.experts.58.down_proj,0.0000013903,0.01000,0.233 11,mlp.experts.59.down_proj,0.0000082734,0.01000,0.232 11,mlp.experts.60.down_proj,0.0000903671,0.01000,0.231 11,mlp.experts.61.down_proj,0.0000072780,0.01000,0.232 11,mlp.experts.62.down_proj,0.0000007346,0.01000,0.233 11,mlp.experts.63.down_proj,0.0000424621,0.01000,0.231 11,mlp.shared_experts.gate_proj,0.0000085321,0.01000,0.350 11,mlp.shared_experts.up_proj,0.0000090716,0.01000,0.336 11,mlp.shared_experts.down_proj,0.0000005094,0.01000,0.496 12,self_attn.kv_a_proj_with_mqa,0.0000058657,0.01000,0.339 12,self_attn.kv_b_proj,0.0001241882,0.01000,0.097 12,self_attn.o_proj,0.0000016597,0.01000,0.342 12,mlp.experts.0.gate_proj,0.0013851634,0.01000,0.346 12,mlp.experts.1.gate_proj,0.0000495527,0.01000,0.335 12,mlp.experts.2.gate_proj,0.0000411436,0.01000,0.335 12,mlp.experts.3.gate_proj,0.0000379087,0.01000,0.337 12,mlp.experts.4.gate_proj,0.0000485509,0.01000,0.357 12,mlp.experts.5.gate_proj,0.0000179119,0.01000,0.337 12,mlp.experts.6.gate_proj,0.0001883261,0.01000,0.336 12,mlp.experts.7.gate_proj,0.0057385032,0.01000,0.335 12,mlp.experts.8.gate_proj,0.0006080249,0.01000,0.344 12,mlp.experts.9.gate_proj,0.0000687967,0.01000,0.350 12,mlp.experts.10.gate_proj,0.0000113901,0.01000,0.353 12,mlp.experts.11.gate_proj,0.0012189870,0.01000,0.336 12,mlp.experts.12.gate_proj,0.0000748386,0.01000,0.335 12,mlp.experts.13.gate_proj,0.0004882555,0.01000,0.346 12,mlp.experts.14.gate_proj,0.0001367856,0.01000,0.348 12,mlp.experts.15.gate_proj,0.0000646253,0.01000,0.336 12,mlp.experts.16.gate_proj,0.0000685647,0.01000,0.336 12,mlp.experts.17.gate_proj,0.0000809248,0.01000,0.337 12,mlp.experts.18.gate_proj,0.0011928664,0.01000,0.337 12,mlp.experts.19.gate_proj,0.0000601179,0.01000,0.337 12,mlp.experts.20.gate_proj,0.0000911689,0.01000,0.337 12,mlp.experts.21.gate_proj,0.0000218162,0.01000,0.340 12,mlp.experts.22.gate_proj,0.0031126309,0.01000,0.336 12,mlp.experts.23.gate_proj,0.0005530911,0.01000,0.339 12,mlp.experts.24.gate_proj,0.0001122665,0.01000,0.338 12,mlp.experts.25.gate_proj,0.0000289726,0.01000,0.336 12,mlp.experts.26.gate_proj,0.0004599349,0.01000,0.336 12,mlp.experts.27.gate_proj,0.0000514014,0.01000,0.339 12,mlp.experts.28.gate_proj,0.0087219404,0.01000,0.339 12,mlp.experts.29.gate_proj,0.0011053770,0.01000,0.335 12,mlp.experts.30.gate_proj,0.0002781949,0.01000,0.338 12,mlp.experts.31.gate_proj,0.0005364320,0.01000,0.340 12,mlp.experts.32.gate_proj,0.0000991016,0.01000,0.335 12,mlp.experts.33.gate_proj,0.0003359020,0.01000,0.337 12,mlp.experts.34.gate_proj,0.0037926070,0.01000,0.338 12,mlp.experts.35.gate_proj,0.0047745269,0.01000,0.336 12,mlp.experts.36.gate_proj,0.0006561153,0.01000,0.337 12,mlp.experts.37.gate_proj,0.0000702797,0.01000,0.338 12,mlp.experts.38.gate_proj,0.0087948239,0.01000,0.339 12,mlp.experts.39.gate_proj,0.0025505351,0.01000,0.335 12,mlp.experts.40.gate_proj,0.0001606003,0.01000,0.339 12,mlp.experts.41.gate_proj,0.0008409719,0.01000,0.336 12,mlp.experts.42.gate_proj,0.0000421933,0.01000,0.333 12,mlp.experts.43.gate_proj,0.0000891302,0.01000,0.337 12,mlp.experts.44.gate_proj,0.0000221485,0.01000,0.342 12,mlp.experts.45.gate_proj,0.0000365155,0.01000,0.335 12,mlp.experts.46.gate_proj,0.0001644137,0.01000,0.336 12,mlp.experts.47.gate_proj,0.0000951846,0.01000,0.337 12,mlp.experts.48.gate_proj,0.0018214044,0.01000,0.334 12,mlp.experts.49.gate_proj,0.0000462898,0.01000,0.334 12,mlp.experts.50.gate_proj,0.0007309486,0.01000,0.335 12,mlp.experts.51.gate_proj,0.0000460083,0.01000,0.334 12,mlp.experts.52.gate_proj,0.0001196330,0.01000,0.335 12,mlp.experts.53.gate_proj,0.0001053090,0.01000,0.336 12,mlp.experts.54.gate_proj,0.0000190670,0.01000,0.336 12,mlp.experts.55.gate_proj,0.0002112181,0.01000,0.336 12,mlp.experts.56.gate_proj,0.0021757276,0.01000,0.337 12,mlp.experts.57.gate_proj,0.0009006563,0.01000,0.336 12,mlp.experts.58.gate_proj,0.0045428582,0.01000,0.335 12,mlp.experts.59.gate_proj,0.0020577912,0.01000,0.336 12,mlp.experts.60.gate_proj,0.0000248896,0.01000,0.336 12,mlp.experts.61.gate_proj,0.0003066716,0.01000,0.335 12,mlp.experts.62.gate_proj,0.0000604229,0.01000,0.336 12,mlp.experts.63.gate_proj,0.0000208005,0.01000,0.336 12,mlp.experts.0.up_proj,0.0013179573,0.01000,0.347 12,mlp.experts.1.up_proj,0.0000458675,0.01000,0.335 12,mlp.experts.2.up_proj,0.0000384147,0.01000,0.336 12,mlp.experts.3.up_proj,0.0000333937,0.01000,0.337 12,mlp.experts.4.up_proj,0.0000444310,0.01000,0.339 12,mlp.experts.5.up_proj,0.0000164061,0.01000,0.334 12,mlp.experts.6.up_proj,0.0001775843,0.01000,0.335 12,mlp.experts.7.up_proj,0.0043040892,0.01000,0.340 12,mlp.experts.8.up_proj,0.0005071508,0.01000,0.341 12,mlp.experts.9.up_proj,0.0000620831,0.01000,0.337 12,mlp.experts.10.up_proj,0.0000088470,0.01000,0.338 12,mlp.experts.11.up_proj,0.0010972577,0.01000,0.337 12,mlp.experts.12.up_proj,0.0000660993,0.01000,0.335 12,mlp.experts.13.up_proj,0.0004345516,0.01000,0.336 12,mlp.experts.14.up_proj,0.0001247340,0.01000,0.334 12,mlp.experts.15.up_proj,0.0000601059,0.01000,0.334 12,mlp.experts.16.up_proj,0.0000558974,0.01000,0.337 12,mlp.experts.17.up_proj,0.0000737127,0.01000,0.337 12,mlp.experts.18.up_proj,0.0010292518,0.01000,0.335 12,mlp.experts.19.up_proj,0.0000551259,0.01000,0.349 12,mlp.experts.20.up_proj,0.0000774951,0.01000,0.345 12,mlp.experts.21.up_proj,0.0000207609,0.01000,0.338 12,mlp.experts.22.up_proj,0.0023876590,0.01000,0.337 12,mlp.experts.23.up_proj,0.0004681274,0.01000,0.338 12,mlp.experts.24.up_proj,0.0000999482,0.01000,0.338 12,mlp.experts.25.up_proj,0.0000264670,0.01000,0.335 12,mlp.experts.26.up_proj,0.0003697442,0.01000,0.336 12,mlp.experts.27.up_proj,0.0000476004,0.01000,0.336 12,mlp.experts.28.up_proj,0.0084784770,0.01000,0.337 12,mlp.experts.29.up_proj,0.0010142665,0.01000,0.336 12,mlp.experts.30.up_proj,0.0002347841,0.01000,0.339 12,mlp.experts.31.up_proj,0.0004669458,0.01000,0.336 12,mlp.experts.32.up_proj,0.0000858615,0.01000,0.334 12,mlp.experts.33.up_proj,0.0003076882,0.01000,0.336 12,mlp.experts.34.up_proj,0.0036724469,0.01000,0.337 12,mlp.experts.35.up_proj,0.0040522567,0.01000,0.340 12,mlp.experts.36.up_proj,0.0005680500,0.01000,0.336 12,mlp.experts.37.up_proj,0.0000620307,0.01000,0.338 12,mlp.experts.38.up_proj,0.0079532411,0.01000,0.336 12,mlp.experts.39.up_proj,0.0018882848,0.01000,0.336 12,mlp.experts.40.up_proj,0.0001528494,0.01000,0.336 12,mlp.experts.41.up_proj,0.0007535598,0.01000,0.337 12,mlp.experts.42.up_proj,0.0000388662,0.01000,0.335 12,mlp.experts.43.up_proj,0.0000830453,0.01000,0.337 12,mlp.experts.44.up_proj,0.0000190919,0.01000,0.337 12,mlp.experts.45.up_proj,0.0000329452,0.01000,0.338 12,mlp.experts.46.up_proj,0.0001465698,0.01000,0.335 12,mlp.experts.47.up_proj,0.0000857702,0.01000,0.337 12,mlp.experts.48.up_proj,0.0014250180,0.01000,0.336 12,mlp.experts.49.up_proj,0.0000436988,0.01000,0.335 12,mlp.experts.50.up_proj,0.0007027620,0.01000,0.336 12,mlp.experts.51.up_proj,0.0000424641,0.01000,0.336 12,mlp.experts.52.up_proj,0.0001051482,0.01000,0.337 12,mlp.experts.53.up_proj,0.0000893663,0.01000,0.337 12,mlp.experts.54.up_proj,0.0000175274,0.01000,0.336 12,mlp.experts.55.up_proj,0.0002041137,0.01000,0.336 12,mlp.experts.56.up_proj,0.0020601064,0.01000,0.334 12,mlp.experts.57.up_proj,0.0006777202,0.01000,0.333 12,mlp.experts.58.up_proj,0.0039923955,0.01000,0.334 12,mlp.experts.59.up_proj,0.0016332012,0.01000,0.334 12,mlp.experts.60.up_proj,0.0000220914,0.01000,0.336 12,mlp.experts.61.up_proj,0.0002770295,0.01000,0.335 12,mlp.experts.62.up_proj,0.0000541595,0.01000,0.335 12,mlp.experts.63.up_proj,0.0000188546,0.01000,0.335 12,mlp.experts.0.down_proj,0.0001156638,0.01000,0.241 12,mlp.experts.1.down_proj,0.0000042280,0.01000,0.229 12,mlp.experts.2.down_proj,0.0000034908,0.01000,0.229 12,mlp.experts.3.down_proj,0.0000032623,0.01000,0.230 12,mlp.experts.4.down_proj,0.0000038789,0.01000,0.229 12,mlp.experts.5.down_proj,0.0000019913,0.01000,0.231 12,mlp.experts.6.down_proj,0.0000198228,0.01000,0.230 12,mlp.experts.7.down_proj,0.0005168147,0.01000,0.231 12,mlp.experts.8.down_proj,0.0000520199,0.01000,0.230 12,mlp.experts.9.down_proj,0.0000084422,0.01000,0.230 12,mlp.experts.10.down_proj,0.0000008171,0.01000,0.230 12,mlp.experts.11.down_proj,0.0000984775,0.01000,0.231 12,mlp.experts.12.down_proj,0.0000055327,0.01000,0.233 12,mlp.experts.13.down_proj,0.0000372233,0.01000,0.234 12,mlp.experts.14.down_proj,0.0000104448,0.01000,0.231 12,mlp.experts.15.down_proj,0.0000066097,0.01000,0.230 12,mlp.experts.16.down_proj,0.0000045849,0.01000,0.231 12,mlp.experts.17.down_proj,0.0000087063,0.01000,0.233 12,mlp.experts.18.down_proj,0.0001168893,0.01000,0.235 12,mlp.experts.19.down_proj,0.0000056700,0.01000,0.232 12,mlp.experts.20.down_proj,0.0000057552,0.01000,0.230 12,mlp.experts.21.down_proj,0.0000018934,0.01000,0.236 12,mlp.experts.22.down_proj,0.0002744359,0.01000,0.231 12,mlp.experts.23.down_proj,0.0000516793,0.01000,0.231 12,mlp.experts.24.down_proj,0.0000092154,0.01000,0.231 12,mlp.experts.25.down_proj,0.0000026246,0.01000,0.230 12,mlp.experts.26.down_proj,0.0000334289,0.01000,0.231 12,mlp.experts.27.down_proj,0.0000045759,0.01000,0.230 12,mlp.experts.28.down_proj,0.0004680299,0.01000,0.238 12,mlp.experts.29.down_proj,0.0001248970,0.01000,0.230 12,mlp.experts.30.down_proj,0.0000197141,0.01000,0.230 12,mlp.experts.31.down_proj,0.0000362260,0.01000,0.230 12,mlp.experts.32.down_proj,0.0000094991,0.01000,0.230 12,mlp.experts.33.down_proj,0.0000257877,0.01000,0.229 12,mlp.experts.34.down_proj,0.0003571889,0.01000,0.229 12,mlp.experts.35.down_proj,0.0004471812,0.01000,0.230 12,mlp.experts.36.down_proj,0.0000638033,0.01000,0.230 12,mlp.experts.37.down_proj,0.0000051554,0.01000,0.230 12,mlp.experts.38.down_proj,0.0007356596,0.01000,0.230 12,mlp.experts.39.down_proj,0.0002279627,0.01000,0.230 12,mlp.experts.40.down_proj,0.0000105577,0.01000,0.229 12,mlp.experts.41.down_proj,0.0000791906,0.01000,0.231 12,mlp.experts.42.down_proj,0.0000037816,0.01000,0.231 12,mlp.experts.43.down_proj,0.0000070770,0.01000,0.231 12,mlp.experts.44.down_proj,0.0000013390,0.01000,0.229 12,mlp.experts.45.down_proj,0.0000036382,0.01000,0.230 12,mlp.experts.46.down_proj,0.0000179277,0.01000,0.230 12,mlp.experts.47.down_proj,0.0000096242,0.01000,0.230 12,mlp.experts.48.down_proj,0.0001529569,0.01000,0.230 12,mlp.experts.49.down_proj,0.0000035192,0.01000,0.231 12,mlp.experts.50.down_proj,0.0000481492,0.01000,0.243 12,mlp.experts.51.down_proj,0.0000030718,0.01000,0.230 12,mlp.experts.52.down_proj,0.0000076943,0.01000,0.229 12,mlp.experts.53.down_proj,0.0000117508,0.01000,0.229 12,mlp.experts.54.down_proj,0.0000019948,0.01000,0.230 12,mlp.experts.55.down_proj,0.0000161263,0.01000,0.229 12,mlp.experts.56.down_proj,0.0002008169,0.01000,0.229 12,mlp.experts.57.down_proj,0.0000839654,0.01000,0.230 12,mlp.experts.58.down_proj,0.0003722253,0.01000,0.230 12,mlp.experts.59.down_proj,0.0001975823,0.01000,0.231 12,mlp.experts.60.down_proj,0.0000016036,0.01000,0.230 12,mlp.experts.61.down_proj,0.0000306009,0.01000,0.230 12,mlp.experts.62.down_proj,0.0000037416,0.01000,0.230 12,mlp.experts.63.down_proj,0.0000015292,0.01000,0.229 12,mlp.shared_experts.gate_proj,0.0000086047,0.01000,0.343 12,mlp.shared_experts.up_proj,0.0000095128,0.01000,0.332 12,mlp.shared_experts.down_proj,0.0000006556,0.01000,0.471 13,self_attn.kv_a_proj_with_mqa,0.0000072822,0.01000,0.335 13,self_attn.kv_b_proj,0.0001040761,0.01000,0.094 13,self_attn.o_proj,0.0000025171,0.01000,0.343 13,mlp.experts.0.gate_proj,0.0000340094,0.01000,0.343 13,mlp.experts.1.gate_proj,0.0002024740,0.01000,0.337 13,mlp.experts.2.gate_proj,0.0002479582,0.01000,0.336 13,mlp.experts.3.gate_proj,0.0000334743,0.01000,0.335 13,mlp.experts.4.gate_proj,0.0000517382,0.01000,0.336 13,mlp.experts.5.gate_proj,0.0000917776,0.01000,0.336 13,mlp.experts.6.gate_proj,0.0000123748,0.01000,0.336 13,mlp.experts.7.gate_proj,0.0001039115,0.01000,0.341 13,mlp.experts.8.gate_proj,0.0000960806,0.01000,0.340 13,mlp.experts.9.gate_proj,0.0000766740,0.01000,0.341 13,mlp.experts.10.gate_proj,0.0007281497,0.01000,0.339 13,mlp.experts.11.gate_proj,0.0030713988,0.01000,0.338 13,mlp.experts.12.gate_proj,0.0001107895,0.01000,0.340 13,mlp.experts.13.gate_proj,0.0001704819,0.01000,0.340 13,mlp.experts.14.gate_proj,0.0000927340,0.01000,0.337 13,mlp.experts.15.gate_proj,0.0000341999,0.01000,0.339 13,mlp.experts.16.gate_proj,0.0001448558,0.01000,0.338 13,mlp.experts.17.gate_proj,0.0001050587,0.01000,0.337 13,mlp.experts.18.gate_proj,0.0000536820,0.01000,0.338 13,mlp.experts.19.gate_proj,0.0000273789,0.01000,0.341 13,mlp.experts.20.gate_proj,0.0010961504,0.01000,0.339 13,mlp.experts.21.gate_proj,0.0003756785,0.01000,0.335 13,mlp.experts.22.gate_proj,0.0010886041,0.01000,0.340 13,mlp.experts.23.gate_proj,0.0000804934,0.01000,0.339 13,mlp.experts.24.gate_proj,0.0005831619,0.01000,0.348 13,mlp.experts.25.gate_proj,0.0000328323,0.01000,0.333 13,mlp.experts.26.gate_proj,0.0020020827,0.01000,0.336 13,mlp.experts.27.gate_proj,0.0005005735,0.01000,0.337 13,mlp.experts.28.gate_proj,0.0003777239,0.01000,0.337 13,mlp.experts.29.gate_proj,0.0001084905,0.01000,0.336 13,mlp.experts.30.gate_proj,0.0007423449,0.01000,0.337 13,mlp.experts.31.gate_proj,0.0000607789,0.01000,0.338 13,mlp.experts.32.gate_proj,0.0004154953,0.01000,0.339 13,mlp.experts.33.gate_proj,0.0002555258,0.01000,0.336 13,mlp.experts.34.gate_proj,0.0001139633,0.01000,0.337 13,mlp.experts.35.gate_proj,0.0014353581,0.01000,0.334 13,mlp.experts.36.gate_proj,0.0000847363,0.01000,0.337 13,mlp.experts.37.gate_proj,0.0007434315,0.01000,0.335 13,mlp.experts.38.gate_proj,0.0000327970,0.01000,0.337 13,mlp.experts.39.gate_proj,0.0005627160,0.01000,0.335 13,mlp.experts.40.gate_proj,0.0000783651,0.01000,0.335 13,mlp.experts.41.gate_proj,0.0005025950,0.01000,0.334 13,mlp.experts.42.gate_proj,0.0040431777,0.01000,0.336 13,mlp.experts.43.gate_proj,0.0000478050,0.01000,0.337 13,mlp.experts.44.gate_proj,0.0005113336,0.01000,0.334 13,mlp.experts.45.gate_proj,0.0000698030,0.01000,0.336 13,mlp.experts.46.gate_proj,0.0000540496,0.01000,0.334 13,mlp.experts.47.gate_proj,0.0000362802,0.01000,0.334 13,mlp.experts.48.gate_proj,0.0009539507,0.01000,0.337 13,mlp.experts.49.gate_proj,0.0002455956,0.01000,0.335 13,mlp.experts.50.gate_proj,0.0000816612,0.01000,0.336 13,mlp.experts.51.gate_proj,0.0000483241,0.01000,0.335 13,mlp.experts.52.gate_proj,0.0000628270,0.01000,0.335 13,mlp.experts.53.gate_proj,0.0001000848,0.01000,0.337 13,mlp.experts.54.gate_proj,0.0000477246,0.01000,0.335 13,mlp.experts.55.gate_proj,0.0000306148,0.01000,0.338 13,mlp.experts.56.gate_proj,0.0019868541,0.01000,0.335 13,mlp.experts.57.gate_proj,0.0002163590,0.01000,0.336 13,mlp.experts.58.gate_proj,0.0012186291,0.01000,0.336 13,mlp.experts.59.gate_proj,0.0004105282,0.01000,0.334 13,mlp.experts.60.gate_proj,0.0041587658,0.01000,0.333 13,mlp.experts.61.gate_proj,0.0001654792,0.01000,0.335 13,mlp.experts.62.gate_proj,0.0000968894,0.01000,0.334 13,mlp.experts.63.gate_proj,0.0000627825,0.01000,0.334 13,mlp.experts.0.up_proj,0.0000299573,0.01000,0.345 13,mlp.experts.1.up_proj,0.0001849993,0.01000,0.335 13,mlp.experts.2.up_proj,0.0002359006,0.01000,0.334 13,mlp.experts.3.up_proj,0.0000310287,0.01000,0.336 13,mlp.experts.4.up_proj,0.0000459030,0.01000,0.335 13,mlp.experts.5.up_proj,0.0000800649,0.01000,0.335 13,mlp.experts.6.up_proj,0.0000086760,0.01000,0.336 13,mlp.experts.7.up_proj,0.0001011184,0.01000,0.337 13,mlp.experts.8.up_proj,0.0000876879,0.01000,0.334 13,mlp.experts.9.up_proj,0.0000694665,0.01000,0.335 13,mlp.experts.10.up_proj,0.0005804925,0.01000,0.336 13,mlp.experts.11.up_proj,0.0028169503,0.01000,0.335 13,mlp.experts.12.up_proj,0.0001037550,0.01000,0.334 13,mlp.experts.13.up_proj,0.0001548165,0.01000,0.338 13,mlp.experts.14.up_proj,0.0000859258,0.01000,0.337 13,mlp.experts.15.up_proj,0.0000281747,0.01000,0.336 13,mlp.experts.16.up_proj,0.0001244052,0.01000,0.334 13,mlp.experts.17.up_proj,0.0000866340,0.01000,0.337 13,mlp.experts.18.up_proj,0.0000474149,0.01000,0.338 13,mlp.experts.19.up_proj,0.0000224260,0.01000,0.336 13,mlp.experts.20.up_proj,0.0009114415,0.01000,0.337 13,mlp.experts.21.up_proj,0.0003275035,0.01000,0.336 13,mlp.experts.22.up_proj,0.0009029629,0.01000,0.336 13,mlp.experts.23.up_proj,0.0000700974,0.01000,0.337 13,mlp.experts.24.up_proj,0.0005287708,0.01000,0.336 13,mlp.experts.25.up_proj,0.0000287316,0.01000,0.334 13,mlp.experts.26.up_proj,0.0017014776,0.01000,0.336 13,mlp.experts.27.up_proj,0.0004420680,0.01000,0.336 13,mlp.experts.28.up_proj,0.0003518576,0.01000,0.333 13,mlp.experts.29.up_proj,0.0000979096,0.01000,0.336 13,mlp.experts.30.up_proj,0.0006719663,0.01000,0.336 13,mlp.experts.31.up_proj,0.0000544074,0.01000,0.336 13,mlp.experts.32.up_proj,0.0003785871,0.01000,0.333 13,mlp.experts.33.up_proj,0.0002055930,0.01000,0.335 13,mlp.experts.34.up_proj,0.0001028144,0.01000,0.336 13,mlp.experts.35.up_proj,0.0011775915,0.01000,0.335 13,mlp.experts.36.up_proj,0.0000745152,0.01000,0.334 13,mlp.experts.37.up_proj,0.0006864203,0.01000,0.336 13,mlp.experts.38.up_proj,0.0000271691,0.01000,0.335 13,mlp.experts.39.up_proj,0.0004557294,0.01000,0.335 13,mlp.experts.40.up_proj,0.0000737900,0.01000,0.338 13,mlp.experts.41.up_proj,0.0004244287,0.01000,0.336 13,mlp.experts.42.up_proj,0.0031079929,0.01000,0.336 13,mlp.experts.43.up_proj,0.0000405498,0.01000,0.341 13,mlp.experts.44.up_proj,0.0004712036,0.01000,0.338 13,mlp.experts.45.up_proj,0.0000566507,0.01000,0.336 13,mlp.experts.46.up_proj,0.0000445765,0.01000,0.335 13,mlp.experts.47.up_proj,0.0000340486,0.01000,0.335 13,mlp.experts.48.up_proj,0.0008008319,0.01000,0.336 13,mlp.experts.49.up_proj,0.0002304222,0.01000,0.333 13,mlp.experts.50.up_proj,0.0000744588,0.01000,0.335 13,mlp.experts.51.up_proj,0.0000448124,0.01000,0.336 13,mlp.experts.52.up_proj,0.0000556637,0.01000,0.336 13,mlp.experts.53.up_proj,0.0000787831,0.01000,0.335 13,mlp.experts.54.up_proj,0.0000440566,0.01000,0.336 13,mlp.experts.55.up_proj,0.0000282258,0.01000,0.334 13,mlp.experts.56.up_proj,0.0016209794,0.01000,0.335 13,mlp.experts.57.up_proj,0.0001792493,0.01000,0.336 13,mlp.experts.58.up_proj,0.0010681011,0.01000,0.337 13,mlp.experts.59.up_proj,0.0003768590,0.01000,0.336 13,mlp.experts.60.up_proj,0.0034875374,0.01000,0.335 13,mlp.experts.61.up_proj,0.0001467033,0.01000,0.337 13,mlp.experts.62.up_proj,0.0000788066,0.01000,0.337 13,mlp.experts.63.up_proj,0.0000562550,0.01000,0.335 13,mlp.experts.0.down_proj,0.0000022769,0.01000,0.240 13,mlp.experts.1.down_proj,0.0000191164,0.01000,0.231 13,mlp.experts.2.down_proj,0.0000263240,0.01000,0.230 13,mlp.experts.3.down_proj,0.0000031684,0.01000,0.229 13,mlp.experts.4.down_proj,0.0000040198,0.01000,0.229 13,mlp.experts.5.down_proj,0.0000118087,0.01000,0.229 13,mlp.experts.6.down_proj,0.0000011566,0.01000,0.229 13,mlp.experts.7.down_proj,0.0000099167,0.01000,0.229 13,mlp.experts.8.down_proj,0.0000095017,0.01000,0.230 13,mlp.experts.9.down_proj,0.0000079066,0.01000,0.230 13,mlp.experts.10.down_proj,0.0000798001,0.01000,0.230 13,mlp.experts.11.down_proj,0.0003467342,0.01000,0.230 13,mlp.experts.12.down_proj,0.0000128580,0.01000,0.232 13,mlp.experts.13.down_proj,0.0000135555,0.01000,0.230 13,mlp.experts.14.down_proj,0.0000088580,0.01000,0.231 13,mlp.experts.15.down_proj,0.0000022582,0.01000,0.230 13,mlp.experts.16.down_proj,0.0000155244,0.01000,0.230 13,mlp.experts.17.down_proj,0.0000165540,0.01000,0.230 13,mlp.experts.18.down_proj,0.0000040007,0.01000,0.231 13,mlp.experts.19.down_proj,0.0000017589,0.01000,0.230 13,mlp.experts.20.down_proj,0.0001188352,0.01000,0.231 13,mlp.experts.21.down_proj,0.0000355038,0.01000,0.230 13,mlp.experts.22.down_proj,0.0001284780,0.01000,0.232 13,mlp.experts.23.down_proj,0.0000086850,0.01000,0.231 13,mlp.experts.24.down_proj,0.0000693222,0.01000,0.230 13,mlp.experts.25.down_proj,0.0000037358,0.01000,0.230 13,mlp.experts.26.down_proj,0.0001622018,0.01000,0.232 13,mlp.experts.27.down_proj,0.0000458956,0.01000,0.230 13,mlp.experts.28.down_proj,0.0000408130,0.01000,0.233 13,mlp.experts.29.down_proj,0.0000092352,0.01000,0.230 13,mlp.experts.30.down_proj,0.0000772746,0.01000,0.231 13,mlp.experts.31.down_proj,0.0000050187,0.01000,0.230 13,mlp.experts.32.down_proj,0.0000413932,0.01000,0.230 13,mlp.experts.33.down_proj,0.0000166095,0.01000,0.231 13,mlp.experts.34.down_proj,0.0000125131,0.01000,0.230 13,mlp.experts.35.down_proj,0.0001821970,0.01000,0.231 13,mlp.experts.36.down_proj,0.0000096776,0.01000,0.232 13,mlp.experts.37.down_proj,0.0000533573,0.01000,0.230 13,mlp.experts.38.down_proj,0.0000023865,0.01000,0.231 13,mlp.experts.39.down_proj,0.0000641804,0.01000,0.230 13,mlp.experts.40.down_proj,0.0000096160,0.01000,0.231 13,mlp.experts.41.down_proj,0.0000373241,0.01000,0.230 13,mlp.experts.42.down_proj,0.0003698671,0.01000,0.230 13,mlp.experts.43.down_proj,0.0000054102,0.01000,0.229 13,mlp.experts.44.down_proj,0.0000417333,0.01000,0.232 13,mlp.experts.45.down_proj,0.0000065484,0.01000,0.232 13,mlp.experts.46.down_proj,0.0000051812,0.01000,0.231 13,mlp.experts.47.down_proj,0.0000038234,0.01000,0.231 13,mlp.experts.48.down_proj,0.0001038422,0.01000,0.232 13,mlp.experts.49.down_proj,0.0000301461,0.01000,0.231 13,mlp.experts.50.down_proj,0.0000064169,0.01000,0.232 13,mlp.experts.51.down_proj,0.0000062555,0.01000,0.232 13,mlp.experts.52.down_proj,0.0000074189,0.01000,0.231 13,mlp.experts.53.down_proj,0.0000084317,0.01000,0.231 13,mlp.experts.54.down_proj,0.0000053995,0.01000,0.234 13,mlp.experts.55.down_proj,0.0000023800,0.01000,0.230 13,mlp.experts.56.down_proj,0.0002015571,0.01000,0.231 13,mlp.experts.57.down_proj,0.0000218112,0.01000,0.229 13,mlp.experts.58.down_proj,0.0001388612,0.01000,0.232 13,mlp.experts.59.down_proj,0.0000398259,0.01000,0.231 13,mlp.experts.60.down_proj,0.0003811030,0.01000,0.232 13,mlp.experts.61.down_proj,0.0000174734,0.01000,0.230 13,mlp.experts.62.down_proj,0.0000077626,0.01000,0.231 13,mlp.experts.63.down_proj,0.0000076527,0.01000,0.229 13,mlp.shared_experts.gate_proj,0.0000096369,0.01000,0.347 13,mlp.shared_experts.up_proj,0.0000110566,0.01000,0.334 13,mlp.shared_experts.down_proj,0.0000008227,0.01000,0.472 14,self_attn.kv_a_proj_with_mqa,0.0000060200,0.01000,0.338 14,self_attn.kv_b_proj,0.0001159971,0.01000,0.095 14,self_attn.o_proj,0.0000016292,0.01000,0.345 14,mlp.experts.0.gate_proj,0.0001744213,0.01000,0.348 14,mlp.experts.1.gate_proj,0.0001011036,0.01000,0.338 14,mlp.experts.2.gate_proj,0.0000915745,0.01000,0.336 14,mlp.experts.3.gate_proj,0.0013322020,0.01000,0.335 14,mlp.experts.4.gate_proj,0.0001109096,0.01000,0.339 14,mlp.experts.5.gate_proj,0.0001011778,0.01000,0.335 14,mlp.experts.6.gate_proj,0.0001523142,0.01000,0.338 14,mlp.experts.7.gate_proj,0.0000649782,0.01000,0.335 14,mlp.experts.8.gate_proj,0.0009856427,0.01000,0.336 14,mlp.experts.9.gate_proj,0.0002004135,0.01000,0.335 14,mlp.experts.10.gate_proj,0.0000537165,0.01000,0.336 14,mlp.experts.11.gate_proj,0.0000295477,0.01000,0.336 14,mlp.experts.12.gate_proj,0.0011943419,0.01000,0.336 14,mlp.experts.13.gate_proj,0.0001662759,0.01000,0.334 14,mlp.experts.14.gate_proj,0.0004062038,0.01000,0.335 14,mlp.experts.15.gate_proj,0.0000534017,0.01000,0.337 14,mlp.experts.16.gate_proj,0.0000722881,0.01000,0.335 14,mlp.experts.17.gate_proj,0.0000294802,0.01000,0.336 14,mlp.experts.18.gate_proj,0.0001891244,0.01000,0.336 14,mlp.experts.19.gate_proj,0.0000299757,0.01000,0.335 14,mlp.experts.20.gate_proj,0.0001375400,0.01000,0.336 14,mlp.experts.21.gate_proj,0.0001057815,0.01000,0.337 14,mlp.experts.22.gate_proj,0.0000372240,0.01000,0.336 14,mlp.experts.23.gate_proj,0.0000203415,0.01000,0.335 14,mlp.experts.24.gate_proj,0.0002766443,0.01000,0.335 14,mlp.experts.25.gate_proj,0.0003246272,0.01000,0.334 14,mlp.experts.26.gate_proj,0.0001075738,0.01000,0.338 14,mlp.experts.27.gate_proj,0.0000868162,0.01000,0.337 14,mlp.experts.28.gate_proj,0.0001188663,0.01000,0.334 14,mlp.experts.29.gate_proj,0.0000250389,0.01000,0.336 14,mlp.experts.30.gate_proj,0.0004410292,0.01000,0.337 14,mlp.experts.31.gate_proj,0.0000765425,0.01000,0.335 14,mlp.experts.32.gate_proj,0.0000546101,0.01000,0.334 14,mlp.experts.33.gate_proj,0.0000571640,0.01000,0.335 14,mlp.experts.34.gate_proj,0.0002746727,0.01000,0.336 14,mlp.experts.35.gate_proj,0.0099147938,0.01000,0.336 14,mlp.experts.36.gate_proj,0.0000565960,0.01000,0.335 14,mlp.experts.37.gate_proj,0.0000821917,0.01000,0.336 14,mlp.experts.38.gate_proj,0.0000464850,0.01000,0.336 14,mlp.experts.39.gate_proj,0.0002746828,0.01000,0.336 14,mlp.experts.40.gate_proj,0.0025838859,0.01000,0.336 14,mlp.experts.41.gate_proj,0.0000565777,0.01000,0.336 14,mlp.experts.42.gate_proj,0.0001640828,0.01000,0.335 14,mlp.experts.43.gate_proj,0.0005226505,0.01000,0.335 14,mlp.experts.44.gate_proj,0.0004252338,0.01000,0.336 14,mlp.experts.45.gate_proj,0.0005646421,0.01000,0.335 14,mlp.experts.46.gate_proj,0.0017503403,0.01000,0.335 14,mlp.experts.47.gate_proj,0.0000874134,0.01000,0.335 14,mlp.experts.48.gate_proj,0.0000933322,0.01000,0.334 14,mlp.experts.49.gate_proj,0.0001604766,0.01000,0.334 14,mlp.experts.50.gate_proj,0.0000675269,0.01000,0.337 14,mlp.experts.51.gate_proj,0.0002142941,0.01000,0.334 14,mlp.experts.52.gate_proj,0.0000823152,0.01000,0.334 14,mlp.experts.53.gate_proj,0.0000478062,0.01000,0.336 14,mlp.experts.54.gate_proj,0.0004859744,0.01000,0.336 14,mlp.experts.55.gate_proj,0.0000354636,0.01000,0.335 14,mlp.experts.56.gate_proj,0.0002271448,0.01000,0.336 14,mlp.experts.57.gate_proj,0.0001700644,0.01000,0.336 14,mlp.experts.58.gate_proj,0.0042142459,0.01000,0.334 14,mlp.experts.59.gate_proj,0.0000701600,0.01000,0.336 14,mlp.experts.60.gate_proj,0.0015824977,0.01000,0.337 14,mlp.experts.61.gate_proj,0.0000982337,0.01000,0.336 14,mlp.experts.62.gate_proj,0.0008196451,0.01000,0.338 14,mlp.experts.63.gate_proj,0.0001686285,0.01000,0.336 14,mlp.experts.0.up_proj,0.0001654743,0.01000,0.348 14,mlp.experts.1.up_proj,0.0000957522,0.01000,0.335 14,mlp.experts.2.up_proj,0.0000832133,0.01000,0.338 14,mlp.experts.3.up_proj,0.0011361202,0.01000,0.335 14,mlp.experts.4.up_proj,0.0000899061,0.01000,0.335 14,mlp.experts.5.up_proj,0.0000911251,0.01000,0.336 14,mlp.experts.6.up_proj,0.0001402294,0.01000,0.336 14,mlp.experts.7.up_proj,0.0000600672,0.01000,0.337 14,mlp.experts.8.up_proj,0.0008321418,0.01000,0.333 14,mlp.experts.9.up_proj,0.0001783918,0.01000,0.335 14,mlp.experts.10.up_proj,0.0000459772,0.01000,0.335 14,mlp.experts.11.up_proj,0.0000272620,0.01000,0.335 14,mlp.experts.12.up_proj,0.0010019011,0.01000,0.337 14,mlp.experts.13.up_proj,0.0001587619,0.01000,0.335 14,mlp.experts.14.up_proj,0.0003569974,0.01000,0.336 14,mlp.experts.15.up_proj,0.0000473760,0.01000,0.335 14,mlp.experts.16.up_proj,0.0000642589,0.01000,0.336 14,mlp.experts.17.up_proj,0.0000265573,0.01000,0.336 14,mlp.experts.18.up_proj,0.0001599818,0.01000,0.337 14,mlp.experts.19.up_proj,0.0000278245,0.01000,0.336 14,mlp.experts.20.up_proj,0.0001159988,0.01000,0.335 14,mlp.experts.21.up_proj,0.0000896912,0.01000,0.335 14,mlp.experts.22.up_proj,0.0000329608,0.01000,0.334 14,mlp.experts.23.up_proj,0.0000178302,0.01000,0.336 14,mlp.experts.24.up_proj,0.0002433974,0.01000,0.335 14,mlp.experts.25.up_proj,0.0003112117,0.01000,0.348 14,mlp.experts.26.up_proj,0.0000935545,0.01000,0.337 14,mlp.experts.27.up_proj,0.0000811075,0.01000,0.336 14,mlp.experts.28.up_proj,0.0001101381,0.01000,0.338 14,mlp.experts.29.up_proj,0.0000225570,0.01000,0.338 14,mlp.experts.30.up_proj,0.0003947296,0.01000,0.339 14,mlp.experts.31.up_proj,0.0000750615,0.01000,0.336 14,mlp.experts.32.up_proj,0.0000491110,0.01000,0.338 14,mlp.experts.33.up_proj,0.0000523193,0.01000,0.340 14,mlp.experts.34.up_proj,0.0002453126,0.01000,0.336 14,mlp.experts.35.up_proj,0.0096147561,0.01000,0.338 14,mlp.experts.36.up_proj,0.0000505594,0.01000,0.337 14,mlp.experts.37.up_proj,0.0000737008,0.01000,0.336 14,mlp.experts.38.up_proj,0.0000409004,0.01000,0.336 14,mlp.experts.39.up_proj,0.0002405987,0.01000,0.338 14,mlp.experts.40.up_proj,0.0024762564,0.01000,0.339 14,mlp.experts.41.up_proj,0.0000518483,0.01000,0.336 14,mlp.experts.42.up_proj,0.0001505250,0.01000,0.339 14,mlp.experts.43.up_proj,0.0004435764,0.01000,0.339 14,mlp.experts.44.up_proj,0.0003700882,0.01000,0.335 14,mlp.experts.45.up_proj,0.0004749478,0.01000,0.335 14,mlp.experts.46.up_proj,0.0015356417,0.01000,0.341 14,mlp.experts.47.up_proj,0.0000806685,0.01000,0.335 14,mlp.experts.48.up_proj,0.0000867901,0.01000,0.337 14,mlp.experts.49.up_proj,0.0001427759,0.01000,0.336 14,mlp.experts.50.up_proj,0.0000604151,0.01000,0.334 14,mlp.experts.51.up_proj,0.0001994964,0.01000,0.334 14,mlp.experts.52.up_proj,0.0000714411,0.01000,0.334 14,mlp.experts.53.up_proj,0.0000431112,0.01000,0.334 14,mlp.experts.54.up_proj,0.0004448272,0.01000,0.332 14,mlp.experts.55.up_proj,0.0000294083,0.01000,0.337 14,mlp.experts.56.up_proj,0.0001895812,0.01000,0.335 14,mlp.experts.57.up_proj,0.0001569355,0.01000,0.336 14,mlp.experts.58.up_proj,0.0038567521,0.01000,0.335 14,mlp.experts.59.up_proj,0.0000598475,0.01000,0.337 14,mlp.experts.60.up_proj,0.0014059640,0.01000,0.334 14,mlp.experts.61.up_proj,0.0000865092,0.01000,0.333 14,mlp.experts.62.up_proj,0.0007187215,0.01000,0.335 14,mlp.experts.63.up_proj,0.0001493908,0.01000,0.337 14,mlp.experts.0.down_proj,0.0000228143,0.01000,0.240 14,mlp.experts.1.down_proj,0.0000079920,0.01000,0.232 14,mlp.experts.2.down_proj,0.0000080059,0.01000,0.233 14,mlp.experts.3.down_proj,0.0001829232,0.01000,0.230 14,mlp.experts.4.down_proj,0.0000103050,0.01000,0.229 14,mlp.experts.5.down_proj,0.0000102942,0.01000,0.231 14,mlp.experts.6.down_proj,0.0000145859,0.01000,0.231 14,mlp.experts.7.down_proj,0.0000072365,0.01000,0.230 14,mlp.experts.8.down_proj,0.0001142714,0.01000,0.232 14,mlp.experts.9.down_proj,0.0000176533,0.01000,0.231 14,mlp.experts.10.down_proj,0.0000045697,0.01000,0.230 14,mlp.experts.11.down_proj,0.0000032599,0.01000,0.231 14,mlp.experts.12.down_proj,0.0001352655,0.01000,0.230 14,mlp.experts.13.down_proj,0.0000173139,0.01000,0.229 14,mlp.experts.14.down_proj,0.0000541584,0.01000,0.229 14,mlp.experts.15.down_proj,0.0000051469,0.01000,0.229 14,mlp.experts.16.down_proj,0.0000062873,0.01000,0.229 14,mlp.experts.17.down_proj,0.0000028733,0.01000,0.230 14,mlp.experts.18.down_proj,0.0000225571,0.01000,0.231 14,mlp.experts.19.down_proj,0.0000032362,0.01000,0.230 14,mlp.experts.20.down_proj,0.0000204162,0.01000,0.239 14,mlp.experts.21.down_proj,0.0000070452,0.01000,0.230 14,mlp.experts.22.down_proj,0.0000042397,0.01000,0.229 14,mlp.experts.23.down_proj,0.0000020569,0.01000,0.230 14,mlp.experts.24.down_proj,0.0000242007,0.01000,0.231 14,mlp.experts.25.down_proj,0.0000414616,0.01000,0.232 14,mlp.experts.26.down_proj,0.0000086533,0.01000,0.230 14,mlp.experts.27.down_proj,0.0000092471,0.01000,0.230 14,mlp.experts.28.down_proj,0.0000111271,0.01000,0.230 14,mlp.experts.29.down_proj,0.0000025050,0.01000,0.230 14,mlp.experts.30.down_proj,0.0000495544,0.01000,0.230 14,mlp.experts.31.down_proj,0.0000078310,0.01000,0.230 14,mlp.experts.32.down_proj,0.0000061395,0.01000,0.230 14,mlp.experts.33.down_proj,0.0000057817,0.01000,0.231 14,mlp.experts.34.down_proj,0.0000279910,0.01000,0.229 14,mlp.experts.35.down_proj,0.0007506876,0.01000,0.230 14,mlp.experts.36.down_proj,0.0000054647,0.01000,0.229 14,mlp.experts.37.down_proj,0.0000075654,0.01000,0.231 14,mlp.experts.38.down_proj,0.0000058946,0.01000,0.237 14,mlp.experts.39.down_proj,0.0000371842,0.01000,0.239 14,mlp.experts.40.down_proj,0.0003179228,0.01000,0.229 14,mlp.experts.41.down_proj,0.0000062600,0.01000,0.229 14,mlp.experts.42.down_proj,0.0000190357,0.01000,0.230 14,mlp.experts.43.down_proj,0.0000421953,0.01000,0.230 14,mlp.experts.44.down_proj,0.0000305686,0.01000,0.229 14,mlp.experts.45.down_proj,0.0000489823,0.01000,0.229 14,mlp.experts.46.down_proj,0.0002006925,0.01000,0.232 14,mlp.experts.47.down_proj,0.0000095941,0.01000,0.230 14,mlp.experts.48.down_proj,0.0000097112,0.01000,0.230 14,mlp.experts.49.down_proj,0.0000149008,0.01000,0.229 14,mlp.experts.50.down_proj,0.0000080187,0.01000,0.229 14,mlp.experts.51.down_proj,0.0000217763,0.01000,0.230 14,mlp.experts.52.down_proj,0.0000079970,0.01000,0.229 14,mlp.experts.53.down_proj,0.0000042243,0.01000,0.229 14,mlp.experts.54.down_proj,0.0000387803,0.01000,0.239 14,mlp.experts.55.down_proj,0.0000023053,0.01000,0.229 14,mlp.experts.56.down_proj,0.0000268623,0.01000,0.230 14,mlp.experts.57.down_proj,0.0000125336,0.01000,0.229 14,mlp.experts.58.down_proj,0.0005009258,0.01000,0.229 14,mlp.experts.59.down_proj,0.0000061770,0.01000,0.229 14,mlp.experts.60.down_proj,0.0001266288,0.01000,0.230 14,mlp.experts.61.down_proj,0.0000098831,0.01000,0.229 14,mlp.experts.62.down_proj,0.0000763607,0.01000,0.230 14,mlp.experts.63.down_proj,0.0000237287,0.01000,0.230 14,mlp.shared_experts.gate_proj,0.0000102314,0.01000,0.341 14,mlp.shared_experts.up_proj,0.0000114315,0.01000,0.333 14,mlp.shared_experts.down_proj,0.0000009380,0.01000,0.471 15,self_attn.kv_a_proj_with_mqa,0.0000056779,0.01000,0.335 15,self_attn.kv_b_proj,0.0000894268,0.01000,0.096 15,self_attn.o_proj,0.0000025472,0.01000,0.346 15,mlp.experts.0.gate_proj,0.0004078218,0.01000,0.347 15,mlp.experts.1.gate_proj,0.0001406248,0.01000,0.337 15,mlp.experts.2.gate_proj,0.0001074388,0.01000,0.335 15,mlp.experts.3.gate_proj,0.0000626361,0.01000,0.334 15,mlp.experts.4.gate_proj,0.0002294598,0.01000,0.335 15,mlp.experts.5.gate_proj,0.0000670663,0.01000,0.333 15,mlp.experts.6.gate_proj,0.0003365097,0.01000,0.335 15,mlp.experts.7.gate_proj,0.0000691515,0.01000,0.335 15,mlp.experts.8.gate_proj,0.0025439165,0.01000,0.339 15,mlp.experts.9.gate_proj,0.0030356428,0.01000,0.334 15,mlp.experts.10.gate_proj,0.0001075349,0.01000,0.336 15,mlp.experts.11.gate_proj,0.0024836795,0.01000,0.336 15,mlp.experts.12.gate_proj,0.0020028239,0.01000,0.335 15,mlp.experts.13.gate_proj,0.0001104895,0.01000,0.335 15,mlp.experts.14.gate_proj,0.0006581256,0.01000,0.336 15,mlp.experts.15.gate_proj,0.0000472757,0.01000,0.335 15,mlp.experts.16.gate_proj,0.0000936148,0.01000,0.336 15,mlp.experts.17.gate_proj,0.0006001933,0.01000,0.338 15,mlp.experts.18.gate_proj,0.0000655916,0.01000,0.336 15,mlp.experts.19.gate_proj,0.0001644913,0.01000,0.334 15,mlp.experts.20.gate_proj,0.0001224072,0.01000,0.339 15,mlp.experts.21.gate_proj,0.0001781665,0.01000,0.337 15,mlp.experts.22.gate_proj,0.0018809798,0.01000,0.336 15,mlp.experts.23.gate_proj,0.0000780691,0.01000,0.336 15,mlp.experts.24.gate_proj,0.0012799012,0.01000,0.336 15,mlp.experts.25.gate_proj,0.0001149476,0.01000,0.335 15,mlp.experts.26.gate_proj,0.0001222360,0.01000,0.335 15,mlp.experts.27.gate_proj,0.0001060574,0.01000,0.339 15,mlp.experts.28.gate_proj,0.0001296382,0.01000,0.336 15,mlp.experts.29.gate_proj,0.0001620870,0.01000,0.336 15,mlp.experts.30.gate_proj,0.0000355810,0.01000,0.338 15,mlp.experts.31.gate_proj,0.0000654110,0.01000,0.338 15,mlp.experts.32.gate_proj,0.0003183749,0.01000,0.337 15,mlp.experts.33.gate_proj,0.0001916473,0.01000,0.339 15,mlp.experts.34.gate_proj,0.0000312648,0.01000,0.340 15,mlp.experts.35.gate_proj,0.0000445108,0.01000,0.335 15,mlp.experts.36.gate_proj,0.0001692117,0.01000,0.337 15,mlp.experts.37.gate_proj,0.0000278481,0.01000,0.337 15,mlp.experts.38.gate_proj,0.0002430515,0.01000,0.338 15,mlp.experts.39.gate_proj,0.0000585729,0.01000,0.336 15,mlp.experts.40.gate_proj,0.0002410725,0.01000,0.338 15,mlp.experts.41.gate_proj,0.0000355373,0.01000,0.337 15,mlp.experts.42.gate_proj,0.0013257115,0.01000,0.337 15,mlp.experts.43.gate_proj,0.0006067439,0.01000,0.337 15,mlp.experts.44.gate_proj,0.0000511404,0.01000,0.338 15,mlp.experts.45.gate_proj,0.0016239223,0.01000,0.339 15,mlp.experts.46.gate_proj,0.0002168815,0.01000,0.335 15,mlp.experts.47.gate_proj,0.0001239627,0.01000,0.334 15,mlp.experts.48.gate_proj,0.0001935554,0.01000,0.334 15,mlp.experts.49.gate_proj,0.0009094123,0.01000,0.337 15,mlp.experts.50.gate_proj,0.0010493458,0.01000,0.337 15,mlp.experts.51.gate_proj,0.0001698201,0.01000,0.337 15,mlp.experts.52.gate_proj,0.0000446625,0.01000,0.335 15,mlp.experts.53.gate_proj,0.0000449176,0.01000,0.350 15,mlp.experts.54.gate_proj,0.0000616420,0.01000,0.334 15,mlp.experts.55.gate_proj,0.0034826788,0.01000,0.334 15,mlp.experts.56.gate_proj,0.0000571969,0.01000,0.335 15,mlp.experts.57.gate_proj,0.0002238377,0.01000,0.336 15,mlp.experts.58.gate_proj,0.0001654684,0.01000,0.335 15,mlp.experts.59.gate_proj,0.0000335040,0.01000,0.335 15,mlp.experts.60.gate_proj,0.0000774272,0.01000,0.336 15,mlp.experts.61.gate_proj,0.0002211560,0.01000,0.335 15,mlp.experts.62.gate_proj,0.0001492062,0.01000,0.337 15,mlp.experts.63.gate_proj,0.0000273634,0.01000,0.336 15,mlp.experts.0.up_proj,0.0003259265,0.01000,0.346 15,mlp.experts.1.up_proj,0.0001311902,0.01000,0.335 15,mlp.experts.2.up_proj,0.0000977386,0.01000,0.338 15,mlp.experts.3.up_proj,0.0000554064,0.01000,0.338 15,mlp.experts.4.up_proj,0.0001914384,0.01000,0.336 15,mlp.experts.5.up_proj,0.0000603773,0.01000,0.335 15,mlp.experts.6.up_proj,0.0003151825,0.01000,0.338 15,mlp.experts.7.up_proj,0.0000589308,0.01000,0.340 15,mlp.experts.8.up_proj,0.0025058200,0.01000,0.337 15,mlp.experts.9.up_proj,0.0025534399,0.01000,0.336 15,mlp.experts.10.up_proj,0.0000979308,0.01000,0.335 15,mlp.experts.11.up_proj,0.0022372065,0.01000,0.336 15,mlp.experts.12.up_proj,0.0018217543,0.01000,0.335 15,mlp.experts.13.up_proj,0.0000984511,0.01000,0.334 15,mlp.experts.14.up_proj,0.0006175467,0.01000,0.333 15,mlp.experts.15.up_proj,0.0000427927,0.01000,0.332 15,mlp.experts.16.up_proj,0.0000804409,0.01000,0.335 15,mlp.experts.17.up_proj,0.0004615410,0.01000,0.355 15,mlp.experts.18.up_proj,0.0000608317,0.01000,0.335 15,mlp.experts.19.up_proj,0.0001408697,0.01000,0.337 15,mlp.experts.20.up_proj,0.0001088838,0.01000,0.334 15,mlp.experts.21.up_proj,0.0001628389,0.01000,0.335 15,mlp.experts.22.up_proj,0.0017773941,0.01000,0.335 15,mlp.experts.23.up_proj,0.0000739930,0.01000,0.341 15,mlp.experts.24.up_proj,0.0011862220,0.01000,0.335 15,mlp.experts.25.up_proj,0.0000951887,0.01000,0.336 15,mlp.experts.26.up_proj,0.0001155561,0.01000,0.337 15,mlp.experts.27.up_proj,0.0000967498,0.01000,0.336 15,mlp.experts.28.up_proj,0.0001140900,0.01000,0.335 15,mlp.experts.29.up_proj,0.0001401683,0.01000,0.335 15,mlp.experts.30.up_proj,0.0000307637,0.01000,0.336 15,mlp.experts.31.up_proj,0.0000566285,0.01000,0.336 15,mlp.experts.32.up_proj,0.0002751786,0.01000,0.335 15,mlp.experts.33.up_proj,0.0001831119,0.01000,0.335 15,mlp.experts.34.up_proj,0.0000273943,0.01000,0.335 15,mlp.experts.35.up_proj,0.0000365861,0.01000,0.334 15,mlp.experts.36.up_proj,0.0001417849,0.01000,0.333 15,mlp.experts.37.up_proj,0.0000240562,0.01000,0.335 15,mlp.experts.38.up_proj,0.0002357954,0.01000,0.334 15,mlp.experts.39.up_proj,0.0000524952,0.01000,0.334 15,mlp.experts.40.up_proj,0.0002113277,0.01000,0.336 15,mlp.experts.41.up_proj,0.0000314112,0.01000,0.333 15,mlp.experts.42.up_proj,0.0011869914,0.01000,0.334 15,mlp.experts.43.up_proj,0.0005232198,0.01000,0.335 15,mlp.experts.44.up_proj,0.0000440798,0.01000,0.335 15,mlp.experts.45.up_proj,0.0012105031,0.01000,0.335 15,mlp.experts.46.up_proj,0.0001977039,0.01000,0.337 15,mlp.experts.47.up_proj,0.0001148218,0.01000,0.337 15,mlp.experts.48.up_proj,0.0001760637,0.01000,0.337 15,mlp.experts.49.up_proj,0.0007986777,0.01000,0.336 15,mlp.experts.50.up_proj,0.0010290477,0.01000,0.335 15,mlp.experts.51.up_proj,0.0001480125,0.01000,0.336 15,mlp.experts.52.up_proj,0.0000372043,0.01000,0.335 15,mlp.experts.53.up_proj,0.0000389718,0.01000,0.335 15,mlp.experts.54.up_proj,0.0000562566,0.01000,0.335 15,mlp.experts.55.up_proj,0.0025664576,0.01000,0.336 15,mlp.experts.56.up_proj,0.0000537859,0.01000,0.335 15,mlp.experts.57.up_proj,0.0001833279,0.01000,0.337 15,mlp.experts.58.up_proj,0.0001432014,0.01000,0.337 15,mlp.experts.59.up_proj,0.0000283068,0.01000,0.337 15,mlp.experts.60.up_proj,0.0000687988,0.01000,0.336 15,mlp.experts.61.up_proj,0.0001761931,0.01000,0.335 15,mlp.experts.62.up_proj,0.0001370098,0.01000,0.336 15,mlp.experts.63.up_proj,0.0000228445,0.01000,0.336 15,mlp.experts.0.down_proj,0.0000485235,0.01000,0.241 15,mlp.experts.1.down_proj,0.0000133957,0.01000,0.231 15,mlp.experts.2.down_proj,0.0000113312,0.01000,0.231 15,mlp.experts.3.down_proj,0.0000073430,0.01000,0.230 15,mlp.experts.4.down_proj,0.0000257662,0.01000,0.230 15,mlp.experts.5.down_proj,0.0000090550,0.01000,0.232 15,mlp.experts.6.down_proj,0.0000472234,0.01000,0.231 15,mlp.experts.7.down_proj,0.0000056803,0.01000,0.229 15,mlp.experts.8.down_proj,0.0002679349,0.01000,0.230 15,mlp.experts.9.down_proj,0.0004562766,0.01000,0.231 15,mlp.experts.10.down_proj,0.0000147181,0.01000,0.231 15,mlp.experts.11.down_proj,0.0002969752,0.01000,0.230 15,mlp.experts.12.down_proj,0.0002536723,0.01000,0.230 15,mlp.experts.13.down_proj,0.0000134016,0.01000,0.230 15,mlp.experts.14.down_proj,0.0001055167,0.01000,0.231 15,mlp.experts.15.down_proj,0.0000055622,0.01000,0.229 15,mlp.experts.16.down_proj,0.0000119017,0.01000,0.231 15,mlp.experts.17.down_proj,0.0000916298,0.01000,0.231 15,mlp.experts.18.down_proj,0.0000078343,0.01000,0.230 15,mlp.experts.19.down_proj,0.0000163447,0.01000,0.231 15,mlp.experts.20.down_proj,0.0000117366,0.01000,0.229 15,mlp.experts.21.down_proj,0.0000291306,0.01000,0.230 15,mlp.experts.22.down_proj,0.0001791188,0.01000,0.230 15,mlp.experts.23.down_proj,0.0000120983,0.01000,0.230 15,mlp.experts.24.down_proj,0.0001167730,0.01000,0.230 15,mlp.experts.25.down_proj,0.0000169651,0.01000,0.229 15,mlp.experts.26.down_proj,0.0000208585,0.01000,0.230 15,mlp.experts.27.down_proj,0.0000135810,0.01000,0.230 15,mlp.experts.28.down_proj,0.0000122850,0.01000,0.231 15,mlp.experts.29.down_proj,0.0000187693,0.01000,0.232 15,mlp.experts.30.down_proj,0.0000030929,0.01000,0.230 15,mlp.experts.31.down_proj,0.0000106479,0.01000,0.230 15,mlp.experts.32.down_proj,0.0000399089,0.01000,0.230 15,mlp.experts.33.down_proj,0.0000239597,0.01000,0.230 15,mlp.experts.34.down_proj,0.0000032414,0.01000,0.231 15,mlp.experts.35.down_proj,0.0000042162,0.01000,0.230 15,mlp.experts.36.down_proj,0.0000252300,0.01000,0.232 15,mlp.experts.37.down_proj,0.0000029777,0.01000,0.232 15,mlp.experts.38.down_proj,0.0000287050,0.01000,0.231 15,mlp.experts.39.down_proj,0.0000081306,0.01000,0.232 15,mlp.experts.40.down_proj,0.0000365658,0.01000,0.230 15,mlp.experts.41.down_proj,0.0000034423,0.01000,0.230 15,mlp.experts.42.down_proj,0.0001423930,0.01000,0.230 15,mlp.experts.43.down_proj,0.0000893533,0.01000,0.231 15,mlp.experts.44.down_proj,0.0000054516,0.01000,0.230 15,mlp.experts.45.down_proj,0.0002535213,0.01000,0.230 15,mlp.experts.46.down_proj,0.0000212248,0.01000,0.231 15,mlp.experts.47.down_proj,0.0000194426,0.01000,0.229 15,mlp.experts.48.down_proj,0.0000216500,0.01000,0.231 15,mlp.experts.49.down_proj,0.0001225260,0.01000,0.231 15,mlp.experts.50.down_proj,0.0001477268,0.01000,0.231 15,mlp.experts.51.down_proj,0.0000204552,0.01000,0.230 15,mlp.experts.52.down_proj,0.0000037196,0.01000,0.230 15,mlp.experts.53.down_proj,0.0000046181,0.01000,0.231 15,mlp.experts.54.down_proj,0.0000082638,0.01000,0.231 15,mlp.experts.55.down_proj,0.0004839538,0.01000,0.231 15,mlp.experts.56.down_proj,0.0000082080,0.01000,0.231 15,mlp.experts.57.down_proj,0.0000188992,0.01000,0.231 15,mlp.experts.58.down_proj,0.0000156513,0.01000,0.234 15,mlp.experts.59.down_proj,0.0000034654,0.01000,0.230 15,mlp.experts.60.down_proj,0.0000090849,0.01000,0.234 15,mlp.experts.61.down_proj,0.0000351365,0.01000,0.232 15,mlp.experts.62.down_proj,0.0000251308,0.01000,0.232 15,mlp.experts.63.down_proj,0.0000032674,0.01000,0.232 15,mlp.shared_experts.gate_proj,0.0000109826,0.01000,0.344 15,mlp.shared_experts.up_proj,0.0000117673,0.01000,0.334 15,mlp.shared_experts.down_proj,0.0000009871,0.01000,0.474 16,self_attn.kv_a_proj_with_mqa,0.0000052921,0.01000,0.338 16,self_attn.kv_b_proj,0.0001560114,0.01000,0.094 16,self_attn.o_proj,0.0000022475,0.01000,0.343 16,mlp.experts.0.gate_proj,0.0000778896,0.01000,0.347 16,mlp.experts.1.gate_proj,0.0001174485,0.01000,0.338 16,mlp.experts.2.gate_proj,0.0001124496,0.01000,0.336 16,mlp.experts.3.gate_proj,0.0007420381,0.01000,0.336 16,mlp.experts.4.gate_proj,0.0000383216,0.01000,0.339 16,mlp.experts.5.gate_proj,0.0002997571,0.01000,0.338 16,mlp.experts.6.gate_proj,0.0000345610,0.01000,0.342 16,mlp.experts.7.gate_proj,0.0001173941,0.01000,0.336 16,mlp.experts.8.gate_proj,0.0000633422,0.01000,0.338 16,mlp.experts.9.gate_proj,0.0005122136,0.01000,0.337 16,mlp.experts.10.gate_proj,0.0004250105,0.01000,0.335 16,mlp.experts.11.gate_proj,0.0000582461,0.01000,0.336 16,mlp.experts.12.gate_proj,0.0006403255,0.01000,0.346 16,mlp.experts.13.gate_proj,0.0016055396,0.01000,0.335 16,mlp.experts.14.gate_proj,0.0000410827,0.01000,0.334 16,mlp.experts.15.gate_proj,0.0001011298,0.01000,0.335 16,mlp.experts.16.gate_proj,0.0000834018,0.01000,0.337 16,mlp.experts.17.gate_proj,0.0000939548,0.01000,0.335 16,mlp.experts.18.gate_proj,0.0003798923,0.01000,0.334 16,mlp.experts.19.gate_proj,0.0004440363,0.01000,0.340 16,mlp.experts.20.gate_proj,0.0001806787,0.01000,0.336 16,mlp.experts.21.gate_proj,0.0001953230,0.01000,0.335 16,mlp.experts.22.gate_proj,0.0001969810,0.01000,0.335 16,mlp.experts.23.gate_proj,0.0000974390,0.01000,0.336 16,mlp.experts.24.gate_proj,0.0000989034,0.01000,0.336 16,mlp.experts.25.gate_proj,0.0000924950,0.01000,0.334 16,mlp.experts.26.gate_proj,0.0003742639,0.01000,0.337 16,mlp.experts.27.gate_proj,0.0000640177,0.01000,0.340 16,mlp.experts.28.gate_proj,0.0011310535,0.01000,0.337 16,mlp.experts.29.gate_proj,0.0000782916,0.01000,0.337 16,mlp.experts.30.gate_proj,0.0000275776,0.01000,0.337 16,mlp.experts.31.gate_proj,0.0009024889,0.01000,0.335 16,mlp.experts.32.gate_proj,0.0003834242,0.01000,0.334 16,mlp.experts.33.gate_proj,0.0001460154,0.01000,0.336 16,mlp.experts.34.gate_proj,0.0010477829,0.01000,0.336 16,mlp.experts.35.gate_proj,0.0001798509,0.01000,0.333 16,mlp.experts.36.gate_proj,0.0000801860,0.01000,0.335 16,mlp.experts.37.gate_proj,0.0001835347,0.01000,0.334 16,mlp.experts.38.gate_proj,0.0002125649,0.01000,0.332 16,mlp.experts.39.gate_proj,0.0003269425,0.01000,0.335 16,mlp.experts.40.gate_proj,0.0028786925,0.01000,0.336 16,mlp.experts.41.gate_proj,0.0000873517,0.01000,0.334 16,mlp.experts.42.gate_proj,0.0002683119,0.01000,0.335 16,mlp.experts.43.gate_proj,0.0002213745,0.01000,0.334 16,mlp.experts.44.gate_proj,0.0003393776,0.01000,0.335 16,mlp.experts.45.gate_proj,0.0007735203,0.01000,0.335 16,mlp.experts.46.gate_proj,0.0002185733,0.01000,0.336 16,mlp.experts.47.gate_proj,0.0001311605,0.01000,0.338 16,mlp.experts.48.gate_proj,0.0000718539,0.01000,0.336 16,mlp.experts.49.gate_proj,0.0004004119,0.01000,0.337 16,mlp.experts.50.gate_proj,0.0000624281,0.01000,0.336 16,mlp.experts.51.gate_proj,0.0008309918,0.01000,0.336 16,mlp.experts.52.gate_proj,0.0000396916,0.01000,0.337 16,mlp.experts.53.gate_proj,0.0022040100,0.01000,0.336 16,mlp.experts.54.gate_proj,0.0001174372,0.01000,0.335 16,mlp.experts.55.gate_proj,0.0000365739,0.01000,0.335 16,mlp.experts.56.gate_proj,0.0001066233,0.01000,0.335 16,mlp.experts.57.gate_proj,0.0000535349,0.01000,0.334 16,mlp.experts.58.gate_proj,0.0004266945,0.01000,0.336 16,mlp.experts.59.gate_proj,0.0001680748,0.01000,0.336 16,mlp.experts.60.gate_proj,0.0001456702,0.01000,0.336 16,mlp.experts.61.gate_proj,0.0000514775,0.01000,0.336 16,mlp.experts.62.gate_proj,0.0001184345,0.01000,0.337 16,mlp.experts.63.gate_proj,0.0001244500,0.01000,0.337 16,mlp.experts.0.up_proj,0.0000710573,0.01000,0.347 16,mlp.experts.1.up_proj,0.0001034518,0.01000,0.337 16,mlp.experts.2.up_proj,0.0000983733,0.01000,0.336 16,mlp.experts.3.up_proj,0.0006262895,0.01000,0.338 16,mlp.experts.4.up_proj,0.0000343144,0.01000,0.334 16,mlp.experts.5.up_proj,0.0002582408,0.01000,0.336 16,mlp.experts.6.up_proj,0.0000312371,0.01000,0.337 16,mlp.experts.7.up_proj,0.0001066101,0.01000,0.339 16,mlp.experts.8.up_proj,0.0000570737,0.01000,0.338 16,mlp.experts.9.up_proj,0.0004151574,0.01000,0.336 16,mlp.experts.10.up_proj,0.0003620899,0.01000,0.336 16,mlp.experts.11.up_proj,0.0000508410,0.01000,0.336 16,mlp.experts.12.up_proj,0.0005590052,0.01000,0.338 16,mlp.experts.13.up_proj,0.0012649197,0.01000,0.336 16,mlp.experts.14.up_proj,0.0000358868,0.01000,0.334 16,mlp.experts.15.up_proj,0.0000920916,0.01000,0.337 16,mlp.experts.16.up_proj,0.0000758734,0.01000,0.340 16,mlp.experts.17.up_proj,0.0000869951,0.01000,0.335 16,mlp.experts.18.up_proj,0.0003125346,0.01000,0.335 16,mlp.experts.19.up_proj,0.0004125304,0.01000,0.337 16,mlp.experts.20.up_proj,0.0001597118,0.01000,0.337 16,mlp.experts.21.up_proj,0.0001544564,0.01000,0.336 16,mlp.experts.22.up_proj,0.0001742895,0.01000,0.336 16,mlp.experts.23.up_proj,0.0000869130,0.01000,0.336 16,mlp.experts.24.up_proj,0.0000914609,0.01000,0.335 16,mlp.experts.25.up_proj,0.0000855010,0.01000,0.335 16,mlp.experts.26.up_proj,0.0003296578,0.01000,0.336 16,mlp.experts.27.up_proj,0.0000579250,0.01000,0.335 16,mlp.experts.28.up_proj,0.0009905749,0.01000,0.334 16,mlp.experts.29.up_proj,0.0000699847,0.01000,0.336 16,mlp.experts.30.up_proj,0.0000225358,0.01000,0.334 16,mlp.experts.31.up_proj,0.0007485063,0.01000,0.334 16,mlp.experts.32.up_proj,0.0003417078,0.01000,0.336 16,mlp.experts.33.up_proj,0.0001311588,0.01000,0.336 16,mlp.experts.34.up_proj,0.0008579120,0.01000,0.338 16,mlp.experts.35.up_proj,0.0001542928,0.01000,0.337 16,mlp.experts.36.up_proj,0.0000700888,0.01000,0.336 16,mlp.experts.37.up_proj,0.0001684834,0.01000,0.337 16,mlp.experts.38.up_proj,0.0001849153,0.01000,0.342 16,mlp.experts.39.up_proj,0.0003002308,0.01000,0.335 16,mlp.experts.40.up_proj,0.0025046441,0.01000,0.337 16,mlp.experts.41.up_proj,0.0000780941,0.01000,0.339 16,mlp.experts.42.up_proj,0.0002351154,0.01000,0.335 16,mlp.experts.43.up_proj,0.0001980208,0.01000,0.339 16,mlp.experts.44.up_proj,0.0002995375,0.01000,0.337 16,mlp.experts.45.up_proj,0.0006029454,0.01000,0.333 16,mlp.experts.46.up_proj,0.0001922175,0.01000,0.335 16,mlp.experts.47.up_proj,0.0001136036,0.01000,0.337 16,mlp.experts.48.up_proj,0.0000588354,0.01000,0.336 16,mlp.experts.49.up_proj,0.0003668266,0.01000,0.334 16,mlp.experts.50.up_proj,0.0000551601,0.01000,0.336 16,mlp.experts.51.up_proj,0.0007645454,0.01000,0.338 16,mlp.experts.52.up_proj,0.0000350743,0.01000,0.337 16,mlp.experts.53.up_proj,0.0020219061,0.01000,0.335 16,mlp.experts.54.up_proj,0.0001065208,0.01000,0.338 16,mlp.experts.55.up_proj,0.0000343807,0.01000,0.337 16,mlp.experts.56.up_proj,0.0000964801,0.01000,0.337 16,mlp.experts.57.up_proj,0.0000472427,0.01000,0.339 16,mlp.experts.58.up_proj,0.0003828444,0.01000,0.340 16,mlp.experts.59.up_proj,0.0001414538,0.01000,0.337 16,mlp.experts.60.up_proj,0.0001287043,0.01000,0.337 16,mlp.experts.61.up_proj,0.0000455666,0.01000,0.338 16,mlp.experts.62.up_proj,0.0001057236,0.01000,0.337 16,mlp.experts.63.up_proj,0.0001060822,0.01000,0.336 16,mlp.experts.0.down_proj,0.0000111976,0.01000,0.244 16,mlp.experts.1.down_proj,0.0000136512,0.01000,0.230 16,mlp.experts.2.down_proj,0.0000165857,0.01000,0.230 16,mlp.experts.3.down_proj,0.0001267618,0.01000,0.230 16,mlp.experts.4.down_proj,0.0000045924,0.01000,0.230 16,mlp.experts.5.down_proj,0.0000531560,0.01000,0.229 16,mlp.experts.6.down_proj,0.0000054265,0.01000,0.230 16,mlp.experts.7.down_proj,0.0000160158,0.01000,0.229 16,mlp.experts.8.down_proj,0.0000100473,0.01000,0.232 16,mlp.experts.9.down_proj,0.0000840853,0.01000,0.230 16,mlp.experts.10.down_proj,0.0000739953,0.01000,0.231 16,mlp.experts.11.down_proj,0.0000092027,0.01000,0.230 16,mlp.experts.12.down_proj,0.0001063254,0.01000,0.231 16,mlp.experts.13.down_proj,0.0002470665,0.01000,0.230 16,mlp.experts.14.down_proj,0.0000042911,0.01000,0.231 16,mlp.experts.15.down_proj,0.0000111648,0.01000,0.230 16,mlp.experts.16.down_proj,0.0000125758,0.01000,0.231 16,mlp.experts.17.down_proj,0.0000141128,0.01000,0.232 16,mlp.experts.18.down_proj,0.0000653120,0.01000,0.230 16,mlp.experts.19.down_proj,0.0000574307,0.01000,0.231 16,mlp.experts.20.down_proj,0.0000312860,0.01000,0.231 16,mlp.experts.21.down_proj,0.0000222082,0.01000,0.243 16,mlp.experts.22.down_proj,0.0000256697,0.01000,0.231 16,mlp.experts.23.down_proj,0.0000187702,0.01000,0.231 16,mlp.experts.24.down_proj,0.0000149665,0.01000,0.231 16,mlp.experts.25.down_proj,0.0000142009,0.01000,0.231 16,mlp.experts.26.down_proj,0.0000576239,0.01000,0.231 16,mlp.experts.27.down_proj,0.0000086749,0.01000,0.231 16,mlp.experts.28.down_proj,0.0001264830,0.01000,0.230 16,mlp.experts.29.down_proj,0.0000105046,0.01000,0.231 16,mlp.experts.30.down_proj,0.0000032584,0.01000,0.230 16,mlp.experts.31.down_proj,0.0001609224,0.01000,0.231 16,mlp.experts.32.down_proj,0.0000713978,0.01000,0.232 16,mlp.experts.33.down_proj,0.0000223223,0.01000,0.235 16,mlp.experts.34.down_proj,0.0001641393,0.01000,0.230 16,mlp.experts.35.down_proj,0.0000199340,0.01000,0.231 16,mlp.experts.36.down_proj,0.0000105451,0.01000,0.229 16,mlp.experts.37.down_proj,0.0000232637,0.01000,0.233 16,mlp.experts.38.down_proj,0.0000273421,0.01000,0.230 16,mlp.experts.39.down_proj,0.0000607193,0.01000,0.235 16,mlp.experts.40.down_proj,0.0004519187,0.01000,0.230 16,mlp.experts.41.down_proj,0.0000102320,0.01000,0.231 16,mlp.experts.42.down_proj,0.0000408810,0.01000,0.231 16,mlp.experts.43.down_proj,0.0000259790,0.01000,0.230 16,mlp.experts.44.down_proj,0.0000432532,0.01000,0.229 16,mlp.experts.45.down_proj,0.0001168734,0.01000,0.229 16,mlp.experts.46.down_proj,0.0000286296,0.01000,0.229 16,mlp.experts.47.down_proj,0.0000235141,0.01000,0.229 16,mlp.experts.48.down_proj,0.0000122844,0.01000,0.229 16,mlp.experts.49.down_proj,0.0000518322,0.01000,0.229 16,mlp.experts.50.down_proj,0.0000080321,0.01000,0.230 16,mlp.experts.51.down_proj,0.0001151745,0.01000,0.229 16,mlp.experts.52.down_proj,0.0000046311,0.01000,0.230 16,mlp.experts.53.down_proj,0.0002671007,0.01000,0.229 16,mlp.experts.54.down_proj,0.0000161865,0.01000,0.230 16,mlp.experts.55.down_proj,0.0000054867,0.01000,0.230 16,mlp.experts.56.down_proj,0.0000145812,0.01000,0.230 16,mlp.experts.57.down_proj,0.0000090811,0.01000,0.230 16,mlp.experts.58.down_proj,0.0000508947,0.01000,0.230 16,mlp.experts.59.down_proj,0.0000284354,0.01000,0.231 16,mlp.experts.60.down_proj,0.0000256963,0.01000,0.232 16,mlp.experts.61.down_proj,0.0000071003,0.01000,0.230 16,mlp.experts.62.down_proj,0.0000176201,0.01000,0.230 16,mlp.experts.63.down_proj,0.0000157619,0.01000,0.230 16,mlp.shared_experts.gate_proj,0.0000135289,0.01000,0.342 16,mlp.shared_experts.up_proj,0.0000128956,0.01000,0.336 16,mlp.shared_experts.down_proj,0.0000011289,0.01000,0.469 17,self_attn.kv_a_proj_with_mqa,0.0000072426,0.01000,0.335 17,self_attn.kv_b_proj,0.0001747303,0.01000,0.095 17,self_attn.o_proj,0.0000028997,0.01000,0.342 17,mlp.experts.0.gate_proj,0.0001810157,0.01000,0.349 17,mlp.experts.1.gate_proj,0.0001008615,0.01000,0.338 17,mlp.experts.2.gate_proj,0.0001894514,0.01000,0.340 17,mlp.experts.3.gate_proj,0.0000626523,0.01000,0.338 17,mlp.experts.4.gate_proj,0.0022195466,0.01000,0.337 17,mlp.experts.5.gate_proj,0.0000522986,0.01000,0.339 17,mlp.experts.6.gate_proj,0.0004317185,0.01000,0.340 17,mlp.experts.7.gate_proj,0.0002557039,0.01000,0.340 17,mlp.experts.8.gate_proj,0.0001738110,0.01000,0.340 17,mlp.experts.9.gate_proj,0.0001194887,0.01000,0.338 17,mlp.experts.10.gate_proj,0.0000674597,0.01000,0.337 17,mlp.experts.11.gate_proj,0.0000904252,0.01000,0.338 17,mlp.experts.12.gate_proj,0.0000987036,0.01000,0.336 17,mlp.experts.13.gate_proj,0.0012005089,0.01000,0.340 17,mlp.experts.14.gate_proj,0.0003512484,0.01000,0.338 17,mlp.experts.15.gate_proj,0.0003537873,0.01000,0.338 17,mlp.experts.16.gate_proj,0.0001404279,0.01000,0.338 17,mlp.experts.17.gate_proj,0.0037810207,0.01000,0.337 17,mlp.experts.18.gate_proj,0.0001352324,0.01000,0.338 17,mlp.experts.19.gate_proj,0.0003917036,0.01000,0.336 17,mlp.experts.20.gate_proj,0.0002420959,0.01000,0.336 17,mlp.experts.21.gate_proj,0.0001245864,0.01000,0.338 17,mlp.experts.22.gate_proj,0.0001179238,0.01000,0.338 17,mlp.experts.23.gate_proj,0.0020469504,0.01000,0.339 17,mlp.experts.24.gate_proj,0.0002329888,0.01000,0.338 17,mlp.experts.25.gate_proj,0.0012145261,0.01000,0.338 17,mlp.experts.26.gate_proj,0.0003803884,0.01000,0.339 17,mlp.experts.27.gate_proj,0.0001577643,0.01000,0.339 17,mlp.experts.28.gate_proj,0.0001085747,0.01000,0.335 17,mlp.experts.29.gate_proj,0.0032046356,0.01000,0.333 17,mlp.experts.30.gate_proj,0.0000619885,0.01000,0.335 17,mlp.experts.31.gate_proj,0.0000872808,0.01000,0.334 17,mlp.experts.32.gate_proj,0.0000554545,0.01000,0.336 17,mlp.experts.33.gate_proj,0.0000736204,0.01000,0.348 17,mlp.experts.34.gate_proj,0.0000585694,0.01000,0.335 17,mlp.experts.35.gate_proj,0.0001399781,0.01000,0.335 17,mlp.experts.36.gate_proj,0.0000786083,0.01000,0.335 17,mlp.experts.37.gate_proj,0.0003179298,0.01000,0.336 17,mlp.experts.38.gate_proj,0.0000919383,0.01000,0.335 17,mlp.experts.39.gate_proj,0.0000941891,0.01000,0.337 17,mlp.experts.40.gate_proj,0.0001272563,0.01000,0.336 17,mlp.experts.41.gate_proj,0.0004558733,0.01000,0.336 17,mlp.experts.42.gate_proj,0.0004972527,0.01000,0.350 17,mlp.experts.43.gate_proj,0.0000590870,0.01000,0.344 17,mlp.experts.44.gate_proj,0.0001005589,0.01000,0.336 17,mlp.experts.45.gate_proj,0.0009026126,0.01000,0.335 17,mlp.experts.46.gate_proj,0.0000931543,0.01000,0.335 17,mlp.experts.47.gate_proj,0.0003824104,0.01000,0.338 17,mlp.experts.48.gate_proj,0.0000720841,0.01000,0.340 17,mlp.experts.49.gate_proj,0.0003788190,0.01000,0.337 17,mlp.experts.50.gate_proj,0.0000816339,0.01000,0.339 17,mlp.experts.51.gate_proj,0.0003392112,0.01000,0.339 17,mlp.experts.52.gate_proj,0.0002643891,0.01000,0.337 17,mlp.experts.53.gate_proj,0.0000748306,0.01000,0.337 17,mlp.experts.54.gate_proj,0.0018975381,0.01000,0.340 17,mlp.experts.55.gate_proj,0.0001456706,0.01000,0.338 17,mlp.experts.56.gate_proj,0.0008819384,0.01000,0.336 17,mlp.experts.57.gate_proj,0.0003955946,0.01000,0.335 17,mlp.experts.58.gate_proj,0.0001367166,0.01000,0.338 17,mlp.experts.59.gate_proj,0.0020040415,0.01000,0.339 17,mlp.experts.60.gate_proj,0.0002465135,0.01000,0.335 17,mlp.experts.61.gate_proj,0.0000546881,0.01000,0.336 17,mlp.experts.62.gate_proj,0.0001077932,0.01000,0.334 17,mlp.experts.63.gate_proj,0.0000537835,0.01000,0.335 17,mlp.experts.0.up_proj,0.0001709189,0.01000,0.348 17,mlp.experts.1.up_proj,0.0000902678,0.01000,0.335 17,mlp.experts.2.up_proj,0.0001641191,0.01000,0.334 17,mlp.experts.3.up_proj,0.0000601278,0.01000,0.338 17,mlp.experts.4.up_proj,0.0017194913,0.01000,0.337 17,mlp.experts.5.up_proj,0.0000479446,0.01000,0.335 17,mlp.experts.6.up_proj,0.0003949156,0.01000,0.338 17,mlp.experts.7.up_proj,0.0002135269,0.01000,0.337 17,mlp.experts.8.up_proj,0.0001577975,0.01000,0.337 17,mlp.experts.9.up_proj,0.0001062274,0.01000,0.335 17,mlp.experts.10.up_proj,0.0000594423,0.01000,0.338 17,mlp.experts.11.up_proj,0.0000813368,0.01000,0.338 17,mlp.experts.12.up_proj,0.0000889273,0.01000,0.337 17,mlp.experts.13.up_proj,0.0010228250,0.01000,0.335 17,mlp.experts.14.up_proj,0.0003262761,0.01000,0.337 17,mlp.experts.15.up_proj,0.0003194439,0.01000,0.337 17,mlp.experts.16.up_proj,0.0001324836,0.01000,0.336 17,mlp.experts.17.up_proj,0.0028175555,0.01000,0.337 17,mlp.experts.18.up_proj,0.0001106614,0.01000,0.340 17,mlp.experts.19.up_proj,0.0003656941,0.01000,0.340 17,mlp.experts.20.up_proj,0.0002168519,0.01000,0.336 17,mlp.experts.21.up_proj,0.0001060688,0.01000,0.338 17,mlp.experts.22.up_proj,0.0001064373,0.01000,0.336 17,mlp.experts.23.up_proj,0.0016782729,0.01000,0.339 17,mlp.experts.24.up_proj,0.0001926745,0.01000,0.336 17,mlp.experts.25.up_proj,0.0011255806,0.01000,0.337 17,mlp.experts.26.up_proj,0.0003081147,0.01000,0.336 17,mlp.experts.27.up_proj,0.0001433576,0.01000,0.336 17,mlp.experts.28.up_proj,0.0000942632,0.01000,0.335 17,mlp.experts.29.up_proj,0.0025999562,0.01000,0.335 17,mlp.experts.30.up_proj,0.0000545120,0.01000,0.335 17,mlp.experts.31.up_proj,0.0000768789,0.01000,0.333 17,mlp.experts.32.up_proj,0.0000523165,0.01000,0.335 17,mlp.experts.33.up_proj,0.0000677242,0.01000,0.336 17,mlp.experts.34.up_proj,0.0000541894,0.01000,0.334 17,mlp.experts.35.up_proj,0.0001243184,0.01000,0.336 17,mlp.experts.36.up_proj,0.0000681685,0.01000,0.336 17,mlp.experts.37.up_proj,0.0002569475,0.01000,0.337 17,mlp.experts.38.up_proj,0.0000838217,0.01000,0.335 17,mlp.experts.39.up_proj,0.0000840334,0.01000,0.336 17,mlp.experts.40.up_proj,0.0001093651,0.01000,0.337 17,mlp.experts.41.up_proj,0.0004043814,0.01000,0.336 17,mlp.experts.42.up_proj,0.0004571876,0.01000,0.337 17,mlp.experts.43.up_proj,0.0000516140,0.01000,0.337 17,mlp.experts.44.up_proj,0.0000887776,0.01000,0.336 17,mlp.experts.45.up_proj,0.0008465732,0.01000,0.334 17,mlp.experts.46.up_proj,0.0000835907,0.01000,0.335 17,mlp.experts.47.up_proj,0.0003364306,0.01000,0.335 17,mlp.experts.48.up_proj,0.0000569978,0.01000,0.333 17,mlp.experts.49.up_proj,0.0003155642,0.01000,0.332 17,mlp.experts.50.up_proj,0.0000730985,0.01000,0.335 17,mlp.experts.51.up_proj,0.0002955254,0.01000,0.335 17,mlp.experts.52.up_proj,0.0002333522,0.01000,0.335 17,mlp.experts.53.up_proj,0.0000675786,0.01000,0.333 17,mlp.experts.54.up_proj,0.0018119955,0.01000,0.333 17,mlp.experts.55.up_proj,0.0001304148,0.01000,0.333 17,mlp.experts.56.up_proj,0.0007298155,0.01000,0.335 17,mlp.experts.57.up_proj,0.0003563748,0.01000,0.335 17,mlp.experts.58.up_proj,0.0001242385,0.01000,0.335 17,mlp.experts.59.up_proj,0.0017623617,0.01000,0.335 17,mlp.experts.60.up_proj,0.0002198216,0.01000,0.334 17,mlp.experts.61.up_proj,0.0000492172,0.01000,0.334 17,mlp.experts.62.up_proj,0.0000989685,0.01000,0.335 17,mlp.experts.63.up_proj,0.0000484439,0.01000,0.334 17,mlp.experts.0.down_proj,0.0000318238,0.01000,0.243 17,mlp.experts.1.down_proj,0.0000191134,0.01000,0.230 17,mlp.experts.2.down_proj,0.0000240892,0.01000,0.229 17,mlp.experts.3.down_proj,0.0000079367,0.01000,0.230 17,mlp.experts.4.down_proj,0.0004698085,0.01000,0.232 17,mlp.experts.5.down_proj,0.0000087791,0.01000,0.231 17,mlp.experts.6.down_proj,0.0000712082,0.01000,0.231 17,mlp.experts.7.down_proj,0.0000538573,0.01000,0.230 17,mlp.experts.8.down_proj,0.0000265979,0.01000,0.231 17,mlp.experts.9.down_proj,0.0000190403,0.01000,0.230 17,mlp.experts.10.down_proj,0.0000099416,0.01000,0.231 17,mlp.experts.11.down_proj,0.0000154137,0.01000,0.231 17,mlp.experts.12.down_proj,0.0000207926,0.01000,0.230 17,mlp.experts.13.down_proj,0.0002300551,0.01000,0.231 17,mlp.experts.14.down_proj,0.0000630040,0.01000,0.232 17,mlp.experts.15.down_proj,0.0000482711,0.01000,0.232 17,mlp.experts.16.down_proj,0.0000247110,0.01000,0.230 17,mlp.experts.17.down_proj,0.0007790224,0.01000,0.231 17,mlp.experts.18.down_proj,0.0000233574,0.01000,0.232 17,mlp.experts.19.down_proj,0.0000682826,0.01000,0.232 17,mlp.experts.20.down_proj,0.0000375481,0.01000,0.230 17,mlp.experts.21.down_proj,0.0000242353,0.01000,0.231 17,mlp.experts.22.down_proj,0.0000213577,0.01000,0.232 17,mlp.experts.23.down_proj,0.0004701005,0.01000,0.231 17,mlp.experts.24.down_proj,0.0000285682,0.01000,0.230 17,mlp.experts.25.down_proj,0.0002045692,0.01000,0.230 17,mlp.experts.26.down_proj,0.0000742866,0.01000,0.229 17,mlp.experts.27.down_proj,0.0000246800,0.01000,0.231 17,mlp.experts.28.down_proj,0.0000170282,0.01000,0.231 17,mlp.experts.29.down_proj,0.0005971124,0.01000,0.229 17,mlp.experts.30.down_proj,0.0000082864,0.01000,0.230 17,mlp.experts.31.down_proj,0.0000130181,0.01000,0.230 17,mlp.experts.32.down_proj,0.0000107618,0.01000,0.239 17,mlp.experts.33.down_proj,0.0000154602,0.01000,0.231 17,mlp.experts.34.down_proj,0.0000089154,0.01000,0.232 17,mlp.experts.35.down_proj,0.0000273204,0.01000,0.230 17,mlp.experts.36.down_proj,0.0000101292,0.01000,0.231 17,mlp.experts.37.down_proj,0.0000622713,0.01000,0.231 17,mlp.experts.38.down_proj,0.0000130420,0.01000,0.232 17,mlp.experts.39.down_proj,0.0000154040,0.01000,0.230 17,mlp.experts.40.down_proj,0.0000237821,0.01000,0.232 17,mlp.experts.41.down_proj,0.0000923275,0.01000,0.231 17,mlp.experts.42.down_proj,0.0000848869,0.01000,0.232 17,mlp.experts.43.down_proj,0.0000089724,0.01000,0.230 17,mlp.experts.44.down_proj,0.0000188118,0.01000,0.230 17,mlp.experts.45.down_proj,0.0001437733,0.01000,0.230 17,mlp.experts.46.down_proj,0.0000137451,0.01000,0.230 17,mlp.experts.47.down_proj,0.0000534847,0.01000,0.230 17,mlp.experts.48.down_proj,0.0000134742,0.01000,0.229 17,mlp.experts.49.down_proj,0.0000541258,0.01000,0.230 17,mlp.experts.50.down_proj,0.0000154972,0.01000,0.230 17,mlp.experts.51.down_proj,0.0000529555,0.01000,0.230 17,mlp.experts.52.down_proj,0.0000432056,0.01000,0.231 17,mlp.experts.53.down_proj,0.0000125426,0.01000,0.232 17,mlp.experts.54.down_proj,0.0003362081,0.01000,0.242 17,mlp.experts.55.down_proj,0.0000256160,0.01000,0.232 17,mlp.experts.56.down_proj,0.0001319080,0.01000,0.232 17,mlp.experts.57.down_proj,0.0000791077,0.01000,0.232 17,mlp.experts.58.down_proj,0.0000192393,0.01000,0.230 17,mlp.experts.59.down_proj,0.0002754436,0.01000,0.230 17,mlp.experts.60.down_proj,0.0000453261,0.01000,0.230 17,mlp.experts.61.down_proj,0.0000074446,0.01000,0.231 17,mlp.experts.62.down_proj,0.0000222946,0.01000,0.230 17,mlp.experts.63.down_proj,0.0000081101,0.01000,0.231 17,mlp.shared_experts.gate_proj,0.0000177419,0.01000,0.348 17,mlp.shared_experts.up_proj,0.0000159067,0.01000,0.344 17,mlp.shared_experts.down_proj,0.0000012289,0.01000,0.472 18,self_attn.kv_a_proj_with_mqa,0.0000068953,0.01000,0.339 18,self_attn.kv_b_proj,0.0001197838,0.01000,0.093 18,self_attn.o_proj,0.0000022479,0.01000,0.348 18,mlp.experts.0.gate_proj,0.0000585655,0.01000,0.344 18,mlp.experts.1.gate_proj,0.0009694422,0.01000,0.340 18,mlp.experts.2.gate_proj,0.0001139910,0.01000,0.339 18,mlp.experts.3.gate_proj,0.0007353886,0.01000,0.340 18,mlp.experts.4.gate_proj,0.0002770201,0.01000,0.335 18,mlp.experts.5.gate_proj,0.0002860246,0.01000,0.337 18,mlp.experts.6.gate_proj,0.0002964773,0.01000,0.339 18,mlp.experts.7.gate_proj,0.0001709369,0.01000,0.338 18,mlp.experts.8.gate_proj,0.0027587119,0.01000,0.335 18,mlp.experts.9.gate_proj,0.0001065625,0.01000,0.336 18,mlp.experts.10.gate_proj,0.0001888731,0.01000,0.340 18,mlp.experts.11.gate_proj,0.0001285019,0.01000,0.339 18,mlp.experts.12.gate_proj,0.0000405103,0.01000,0.336 18,mlp.experts.13.gate_proj,0.0000506481,0.01000,0.337 18,mlp.experts.14.gate_proj,0.0004985882,0.01000,0.341 18,mlp.experts.15.gate_proj,0.0006330207,0.01000,0.337 18,mlp.experts.16.gate_proj,0.0008901184,0.01000,0.336 18,mlp.experts.17.gate_proj,0.0003913988,0.01000,0.336 18,mlp.experts.18.gate_proj,0.0048111778,0.01000,0.337 18,mlp.experts.19.gate_proj,0.0001917997,0.01000,0.333 18,mlp.experts.20.gate_proj,0.0000959407,0.01000,0.335 18,mlp.experts.21.gate_proj,0.0003565126,0.01000,0.334 18,mlp.experts.22.gate_proj,0.0000585908,0.01000,0.335 18,mlp.experts.23.gate_proj,0.0002224060,0.01000,0.336 18,mlp.experts.24.gate_proj,0.0002285316,0.01000,0.335 18,mlp.experts.25.gate_proj,0.0006886663,0.01000,0.336 18,mlp.experts.26.gate_proj,0.0000709516,0.01000,0.337 18,mlp.experts.27.gate_proj,0.0000991519,0.01000,0.336 18,mlp.experts.28.gate_proj,0.0022712029,0.01000,0.335 18,mlp.experts.29.gate_proj,0.0005192473,0.01000,0.335 18,mlp.experts.30.gate_proj,0.0001192203,0.01000,0.336 18,mlp.experts.31.gate_proj,0.0001472446,0.01000,0.337 18,mlp.experts.32.gate_proj,0.0001443296,0.01000,0.335 18,mlp.experts.33.gate_proj,0.0001234793,0.01000,0.337 18,mlp.experts.34.gate_proj,0.0004824311,0.01000,0.337 18,mlp.experts.35.gate_proj,0.0008620100,0.01000,0.338 18,mlp.experts.36.gate_proj,0.0001245052,0.01000,0.337 18,mlp.experts.37.gate_proj,0.0002825948,0.01000,0.335 18,mlp.experts.38.gate_proj,0.0000706972,0.01000,0.335 18,mlp.experts.39.gate_proj,0.0001011752,0.01000,0.335 18,mlp.experts.40.gate_proj,0.0002186770,0.01000,0.334 18,mlp.experts.41.gate_proj,0.0002852596,0.01000,0.335 18,mlp.experts.42.gate_proj,0.0000490451,0.01000,0.334 18,mlp.experts.43.gate_proj,0.0001417816,0.01000,0.335 18,mlp.experts.44.gate_proj,0.0011726479,0.01000,0.333 18,mlp.experts.45.gate_proj,0.0000708501,0.01000,0.334 18,mlp.experts.46.gate_proj,0.0000430900,0.01000,0.335 18,mlp.experts.47.gate_proj,0.0001493059,0.01000,0.335 18,mlp.experts.48.gate_proj,0.0001704573,0.01000,0.336 18,mlp.experts.49.gate_proj,0.0001049410,0.01000,0.334 18,mlp.experts.50.gate_proj,0.0007439299,0.01000,0.334 18,mlp.experts.51.gate_proj,0.0003827808,0.01000,0.334 18,mlp.experts.52.gate_proj,0.0028036427,0.01000,0.334 18,mlp.experts.53.gate_proj,0.0001228828,0.01000,0.335 18,mlp.experts.54.gate_proj,0.0012573127,0.01000,0.335 18,mlp.experts.55.gate_proj,0.0004564335,0.01000,0.335 18,mlp.experts.56.gate_proj,0.0002014203,0.01000,0.334 18,mlp.experts.57.gate_proj,0.0003601344,0.01000,0.334 18,mlp.experts.58.gate_proj,0.0001517240,0.01000,0.334 18,mlp.experts.59.gate_proj,0.0001336019,0.01000,0.334 18,mlp.experts.60.gate_proj,0.0014744302,0.01000,0.334 18,mlp.experts.61.gate_proj,0.0000387593,0.01000,0.334 18,mlp.experts.62.gate_proj,0.0010240757,0.01000,0.334 18,mlp.experts.63.gate_proj,0.0003291321,0.01000,0.333 18,mlp.experts.0.up_proj,0.0000520310,0.01000,0.348 18,mlp.experts.1.up_proj,0.0008794230,0.01000,0.336 18,mlp.experts.2.up_proj,0.0001035634,0.01000,0.335 18,mlp.experts.3.up_proj,0.0007073793,0.01000,0.337 18,mlp.experts.4.up_proj,0.0002574595,0.01000,0.336 18,mlp.experts.5.up_proj,0.0002622692,0.01000,0.336 18,mlp.experts.6.up_proj,0.0002704230,0.01000,0.337 18,mlp.experts.7.up_proj,0.0001545404,0.01000,0.336 18,mlp.experts.8.up_proj,0.0023630994,0.01000,0.336 18,mlp.experts.9.up_proj,0.0000908027,0.01000,0.336 18,mlp.experts.10.up_proj,0.0001590856,0.01000,0.336 18,mlp.experts.11.up_proj,0.0001210466,0.01000,0.334 18,mlp.experts.12.up_proj,0.0000364165,0.01000,0.336 18,mlp.experts.13.up_proj,0.0000454179,0.01000,0.336 18,mlp.experts.14.up_proj,0.0004169423,0.01000,0.335 18,mlp.experts.15.up_proj,0.0005498246,0.01000,0.336 18,mlp.experts.16.up_proj,0.0007897862,0.01000,0.335 18,mlp.experts.17.up_proj,0.0003658962,0.01000,0.337 18,mlp.experts.18.up_proj,0.0043742533,0.01000,0.336 18,mlp.experts.19.up_proj,0.0001622611,0.01000,0.339 18,mlp.experts.20.up_proj,0.0000887196,0.01000,0.334 18,mlp.experts.21.up_proj,0.0002999062,0.01000,0.336 18,mlp.experts.22.up_proj,0.0000534555,0.01000,0.337 18,mlp.experts.23.up_proj,0.0002054262,0.01000,0.335 18,mlp.experts.24.up_proj,0.0001971449,0.01000,0.334 18,mlp.experts.25.up_proj,0.0005804809,0.01000,0.337 18,mlp.experts.26.up_proj,0.0000628743,0.01000,0.335 18,mlp.experts.27.up_proj,0.0000827847,0.01000,0.335 18,mlp.experts.28.up_proj,0.0019250023,0.01000,0.335 18,mlp.experts.29.up_proj,0.0004856421,0.01000,0.336 18,mlp.experts.30.up_proj,0.0001048129,0.01000,0.334 18,mlp.experts.31.up_proj,0.0001298771,0.01000,0.336 18,mlp.experts.32.up_proj,0.0001356833,0.01000,0.335 18,mlp.experts.33.up_proj,0.0001113501,0.01000,0.335 18,mlp.experts.34.up_proj,0.0004283150,0.01000,0.334 18,mlp.experts.35.up_proj,0.0007920725,0.01000,0.333 18,mlp.experts.36.up_proj,0.0001099391,0.01000,0.339 18,mlp.experts.37.up_proj,0.0002634051,0.01000,0.337 18,mlp.experts.38.up_proj,0.0000631884,0.01000,0.336 18,mlp.experts.39.up_proj,0.0000909375,0.01000,0.338 18,mlp.experts.40.up_proj,0.0002000378,0.01000,0.335 18,mlp.experts.41.up_proj,0.0002612362,0.01000,0.336 18,mlp.experts.42.up_proj,0.0000412921,0.01000,0.335 18,mlp.experts.43.up_proj,0.0001281795,0.01000,0.335 18,mlp.experts.44.up_proj,0.0009782914,0.01000,0.334 18,mlp.experts.45.up_proj,0.0000641397,0.01000,0.336 18,mlp.experts.46.up_proj,0.0000380612,0.01000,0.337 18,mlp.experts.47.up_proj,0.0001377938,0.01000,0.336 18,mlp.experts.48.up_proj,0.0001555563,0.01000,0.334 18,mlp.experts.49.up_proj,0.0000866093,0.01000,0.335 18,mlp.experts.50.up_proj,0.0006375672,0.01000,0.337 18,mlp.experts.51.up_proj,0.0003422609,0.01000,0.336 18,mlp.experts.52.up_proj,0.0024688274,0.01000,0.336 18,mlp.experts.53.up_proj,0.0001106872,0.01000,0.336 18,mlp.experts.54.up_proj,0.0011260849,0.01000,0.335 18,mlp.experts.55.up_proj,0.0003980027,0.01000,0.336 18,mlp.experts.56.up_proj,0.0001727766,0.01000,0.336 18,mlp.experts.57.up_proj,0.0003168213,0.01000,0.337 18,mlp.experts.58.up_proj,0.0001372437,0.01000,0.337 18,mlp.experts.59.up_proj,0.0001161497,0.01000,0.338 18,mlp.experts.60.up_proj,0.0012614044,0.01000,0.339 18,mlp.experts.61.up_proj,0.0000354255,0.01000,0.338 18,mlp.experts.62.up_proj,0.0008694247,0.01000,0.336 18,mlp.experts.63.up_proj,0.0002796153,0.01000,0.336 18,mlp.experts.0.down_proj,0.0000128132,0.01000,0.243 18,mlp.experts.1.down_proj,0.0001587849,0.01000,0.230 18,mlp.experts.2.down_proj,0.0000212120,0.01000,0.231 18,mlp.experts.3.down_proj,0.0001898121,0.01000,0.230 18,mlp.experts.4.down_proj,0.0000583454,0.01000,0.230 18,mlp.experts.5.down_proj,0.0000442144,0.01000,0.229 18,mlp.experts.6.down_proj,0.0000583023,0.01000,0.230 18,mlp.experts.7.down_proj,0.0000300091,0.01000,0.230 18,mlp.experts.8.down_proj,0.0006101688,0.01000,0.229 18,mlp.experts.9.down_proj,0.0000169986,0.01000,0.232 18,mlp.experts.10.down_proj,0.0000371831,0.01000,0.230 18,mlp.experts.11.down_proj,0.0000263397,0.01000,0.230 18,mlp.experts.12.down_proj,0.0000070534,0.01000,0.230 18,mlp.experts.13.down_proj,0.0000088388,0.01000,0.230 18,mlp.experts.14.down_proj,0.0001098604,0.01000,0.230 18,mlp.experts.15.down_proj,0.0001062497,0.01000,0.230 18,mlp.experts.16.down_proj,0.0001489731,0.01000,0.231 18,mlp.experts.17.down_proj,0.0000675234,0.01000,0.231 18,mlp.experts.18.down_proj,0.0008802969,0.01000,0.231 18,mlp.experts.19.down_proj,0.0000334111,0.01000,0.230 18,mlp.experts.20.down_proj,0.0000190065,0.01000,0.230 18,mlp.experts.21.down_proj,0.0000607946,0.01000,0.234 18,mlp.experts.22.down_proj,0.0000130778,0.01000,0.231 18,mlp.experts.23.down_proj,0.0000421636,0.01000,0.233 18,mlp.experts.24.down_proj,0.0000479594,0.01000,0.231 18,mlp.experts.25.down_proj,0.0001273030,0.01000,0.230 18,mlp.experts.26.down_proj,0.0000160004,0.01000,0.231 18,mlp.experts.27.down_proj,0.0000152543,0.01000,0.231 18,mlp.experts.28.down_proj,0.0003814460,0.01000,0.238 18,mlp.experts.29.down_proj,0.0001117323,0.01000,0.231 18,mlp.experts.30.down_proj,0.0000207242,0.01000,0.232 18,mlp.experts.31.down_proj,0.0000289890,0.01000,0.232 18,mlp.experts.32.down_proj,0.0000367333,0.01000,0.232 18,mlp.experts.33.down_proj,0.0000254921,0.01000,0.235 18,mlp.experts.34.down_proj,0.0000989137,0.01000,0.231 18,mlp.experts.35.down_proj,0.0001894335,0.01000,0.232 18,mlp.experts.36.down_proj,0.0000208253,0.01000,0.232 18,mlp.experts.37.down_proj,0.0000523476,0.01000,0.232 18,mlp.experts.38.down_proj,0.0000144147,0.01000,0.232 18,mlp.experts.39.down_proj,0.0000185575,0.01000,0.231 18,mlp.experts.40.down_proj,0.0000379503,0.01000,0.232 18,mlp.experts.41.down_proj,0.0000513303,0.01000,0.230 18,mlp.experts.42.down_proj,0.0000093914,0.01000,0.232 18,mlp.experts.43.down_proj,0.0000250579,0.01000,0.232 18,mlp.experts.44.down_proj,0.0002635244,0.01000,0.230 18,mlp.experts.45.down_proj,0.0000160775,0.01000,0.231 18,mlp.experts.46.down_proj,0.0000090146,0.01000,0.230 18,mlp.experts.47.down_proj,0.0000303989,0.01000,0.231 18,mlp.experts.48.down_proj,0.0000373926,0.01000,0.230 18,mlp.experts.49.down_proj,0.0000166320,0.01000,0.230 18,mlp.experts.50.down_proj,0.0001243282,0.01000,0.231 18,mlp.experts.51.down_proj,0.0000529661,0.01000,0.231 18,mlp.experts.52.down_proj,0.0005411969,0.01000,0.231 18,mlp.experts.53.down_proj,0.0000279570,0.01000,0.252 18,mlp.experts.54.down_proj,0.0002147544,0.01000,0.231 18,mlp.experts.55.down_proj,0.0000952831,0.01000,0.230 18,mlp.experts.56.down_proj,0.0000456239,0.01000,0.230 18,mlp.experts.57.down_proj,0.0000857419,0.01000,0.231 18,mlp.experts.58.down_proj,0.0000355759,0.01000,0.230 18,mlp.experts.59.down_proj,0.0000226295,0.01000,0.230 18,mlp.experts.60.down_proj,0.0002957827,0.01000,0.229 18,mlp.experts.61.down_proj,0.0000078555,0.01000,0.231 18,mlp.experts.62.down_proj,0.0002044852,0.01000,0.229 18,mlp.experts.63.down_proj,0.0000721038,0.01000,0.231 18,mlp.shared_experts.gate_proj,0.0000197168,0.01000,0.343 18,mlp.shared_experts.up_proj,0.0000165643,0.01000,0.332 18,mlp.shared_experts.down_proj,0.0000015798,0.01000,0.475 19,self_attn.kv_a_proj_with_mqa,0.0000059814,0.01000,0.348 19,self_attn.kv_b_proj,0.0001500018,0.01000,0.096 19,self_attn.o_proj,0.0000031563,0.01000,0.344 19,mlp.experts.0.gate_proj,0.0002239492,0.01000,0.351 19,mlp.experts.1.gate_proj,0.0002173487,0.01000,0.340 19,mlp.experts.2.gate_proj,0.0002036890,0.01000,0.339 19,mlp.experts.3.gate_proj,0.0003130217,0.01000,0.349 19,mlp.experts.4.gate_proj,0.0003828197,0.01000,0.338 19,mlp.experts.5.gate_proj,0.0001023224,0.01000,0.342 19,mlp.experts.6.gate_proj,0.0001144241,0.01000,0.346 19,mlp.experts.7.gate_proj,0.0018990166,0.01000,0.338 19,mlp.experts.8.gate_proj,0.0001749948,0.01000,0.337 19,mlp.experts.9.gate_proj,0.0013692044,0.01000,0.337 19,mlp.experts.10.gate_proj,0.0001122087,0.01000,0.338 19,mlp.experts.11.gate_proj,0.0001409629,0.01000,0.335 19,mlp.experts.12.gate_proj,0.0002403095,0.01000,0.341 19,mlp.experts.13.gate_proj,0.0007177668,0.01000,0.350 19,mlp.experts.14.gate_proj,0.0000916252,0.01000,0.341 19,mlp.experts.15.gate_proj,0.0002457592,0.01000,0.337 19,mlp.experts.16.gate_proj,0.0002397189,0.01000,0.339 19,mlp.experts.17.gate_proj,0.0001511897,0.01000,0.335 19,mlp.experts.18.gate_proj,0.0001904021,0.01000,0.337 19,mlp.experts.19.gate_proj,0.0022698628,0.01000,0.337 19,mlp.experts.20.gate_proj,0.0003582578,0.01000,0.339 19,mlp.experts.21.gate_proj,0.0003046575,0.01000,0.336 19,mlp.experts.22.gate_proj,0.0004126314,0.01000,0.337 19,mlp.experts.23.gate_proj,0.0002108016,0.01000,0.335 19,mlp.experts.24.gate_proj,0.0019855049,0.01000,0.338 19,mlp.experts.25.gate_proj,0.0002106488,0.01000,0.336 19,mlp.experts.26.gate_proj,0.0000906962,0.01000,0.336 19,mlp.experts.27.gate_proj,0.0001106348,0.01000,0.336 19,mlp.experts.28.gate_proj,0.0001966163,0.01000,0.337 19,mlp.experts.29.gate_proj,0.0000916305,0.01000,0.336 19,mlp.experts.30.gate_proj,0.0003787413,0.01000,0.336 19,mlp.experts.31.gate_proj,0.0003640518,0.01000,0.337 19,mlp.experts.32.gate_proj,0.0001822089,0.01000,0.336 19,mlp.experts.33.gate_proj,0.0007165184,0.01000,0.334 19,mlp.experts.34.gate_proj,0.0004148041,0.01000,0.339 19,mlp.experts.35.gate_proj,0.0001100490,0.01000,0.341 19,mlp.experts.36.gate_proj,0.0019080062,0.01000,0.338 19,mlp.experts.37.gate_proj,0.0002599715,0.01000,0.337 19,mlp.experts.38.gate_proj,0.0007509307,0.01000,0.336 19,mlp.experts.39.gate_proj,0.0000551948,0.01000,0.338 19,mlp.experts.40.gate_proj,0.0002037437,0.01000,0.340 19,mlp.experts.41.gate_proj,0.0001511664,0.01000,0.340 19,mlp.experts.42.gate_proj,0.0002155834,0.01000,0.340 19,mlp.experts.43.gate_proj,0.0001299902,0.01000,0.337 19,mlp.experts.44.gate_proj,0.0002973253,0.01000,0.337 19,mlp.experts.45.gate_proj,0.0002590200,0.01000,0.337 19,mlp.experts.46.gate_proj,0.0001893372,0.01000,0.338 19,mlp.experts.47.gate_proj,0.0001607948,0.01000,0.339 19,mlp.experts.48.gate_proj,0.0000697633,0.01000,0.337 19,mlp.experts.49.gate_proj,0.0301082775,0.01000,0.337 19,mlp.experts.50.gate_proj,0.0000873931,0.01000,0.350 19,mlp.experts.51.gate_proj,0.0002748685,0.01000,0.339 19,mlp.experts.52.gate_proj,0.0001936103,0.01000,0.340 19,mlp.experts.53.gate_proj,0.0003591292,0.01000,0.338 19,mlp.experts.54.gate_proj,0.0000742171,0.01000,0.336 19,mlp.experts.55.gate_proj,0.0001434023,0.01000,0.340 19,mlp.experts.56.gate_proj,0.0007110969,0.01000,0.335 19,mlp.experts.57.gate_proj,0.0001843158,0.01000,0.337 19,mlp.experts.58.gate_proj,0.0001286802,0.01000,0.335 19,mlp.experts.59.gate_proj,0.0001077514,0.01000,0.335 19,mlp.experts.60.gate_proj,0.0005511871,0.01000,0.334 19,mlp.experts.61.gate_proj,0.0001345779,0.01000,0.335 19,mlp.experts.62.gate_proj,0.0001067479,0.01000,0.335 19,mlp.experts.63.gate_proj,0.0001016764,0.01000,0.335 19,mlp.experts.0.up_proj,0.0001876012,0.01000,0.348 19,mlp.experts.1.up_proj,0.0001934901,0.01000,0.339 19,mlp.experts.2.up_proj,0.0001854619,0.01000,0.341 19,mlp.experts.3.up_proj,0.0002537046,0.01000,0.339 19,mlp.experts.4.up_proj,0.0003411589,0.01000,0.339 19,mlp.experts.5.up_proj,0.0000944049,0.01000,0.338 19,mlp.experts.6.up_proj,0.0000991998,0.01000,0.337 19,mlp.experts.7.up_proj,0.0016649996,0.01000,0.336 19,mlp.experts.8.up_proj,0.0001529221,0.01000,0.340 19,mlp.experts.9.up_proj,0.0011825428,0.01000,0.339 19,mlp.experts.10.up_proj,0.0001037113,0.01000,0.337 19,mlp.experts.11.up_proj,0.0001288118,0.01000,0.338 19,mlp.experts.12.up_proj,0.0002204221,0.01000,0.335 19,mlp.experts.13.up_proj,0.0006432895,0.01000,0.338 19,mlp.experts.14.up_proj,0.0000768381,0.01000,0.338 19,mlp.experts.15.up_proj,0.0002222239,0.01000,0.338 19,mlp.experts.16.up_proj,0.0002133852,0.01000,0.334 19,mlp.experts.17.up_proj,0.0001361059,0.01000,0.335 19,mlp.experts.18.up_proj,0.0001686748,0.01000,0.336 19,mlp.experts.19.up_proj,0.0020325159,0.01000,0.335 19,mlp.experts.20.up_proj,0.0003078940,0.01000,0.337 19,mlp.experts.21.up_proj,0.0002689208,0.01000,0.338 19,mlp.experts.22.up_proj,0.0003685879,0.01000,0.336 19,mlp.experts.23.up_proj,0.0001863187,0.01000,0.337 19,mlp.experts.24.up_proj,0.0016802727,0.01000,0.338 19,mlp.experts.25.up_proj,0.0001687468,0.01000,0.337 19,mlp.experts.26.up_proj,0.0000820337,0.01000,0.338 19,mlp.experts.27.up_proj,0.0000993970,0.01000,0.336 19,mlp.experts.28.up_proj,0.0001779919,0.01000,0.337 19,mlp.experts.29.up_proj,0.0000814615,0.01000,0.337 19,mlp.experts.30.up_proj,0.0003531803,0.01000,0.338 19,mlp.experts.31.up_proj,0.0003273552,0.01000,0.344 19,mlp.experts.32.up_proj,0.0001698777,0.01000,0.345 19,mlp.experts.33.up_proj,0.0006398317,0.01000,0.336 19,mlp.experts.34.up_proj,0.0003781436,0.01000,0.335 19,mlp.experts.35.up_proj,0.0001034109,0.01000,0.337 19,mlp.experts.36.up_proj,0.0016617535,0.01000,0.335 19,mlp.experts.37.up_proj,0.0002311927,0.01000,0.336 19,mlp.experts.38.up_proj,0.0006773841,0.01000,0.338 19,mlp.experts.39.up_proj,0.0000497201,0.01000,0.336 19,mlp.experts.40.up_proj,0.0001916979,0.01000,0.335 19,mlp.experts.41.up_proj,0.0001263543,0.01000,0.336 19,mlp.experts.42.up_proj,0.0001978635,0.01000,0.337 19,mlp.experts.43.up_proj,0.0001180604,0.01000,0.337 19,mlp.experts.44.up_proj,0.0002651488,0.01000,0.338 19,mlp.experts.45.up_proj,0.0002396606,0.01000,0.336 19,mlp.experts.46.up_proj,0.0001755380,0.01000,0.337 19,mlp.experts.47.up_proj,0.0001401006,0.01000,0.336 19,mlp.experts.48.up_proj,0.0000650677,0.01000,0.335 19,mlp.experts.49.up_proj,0.0243318845,0.01000,0.336 19,mlp.experts.50.up_proj,0.0000781466,0.01000,0.334 19,mlp.experts.51.up_proj,0.0002387910,0.01000,0.334 19,mlp.experts.52.up_proj,0.0001746457,0.01000,0.334 19,mlp.experts.53.up_proj,0.0003282277,0.01000,0.336 19,mlp.experts.54.up_proj,0.0000682780,0.01000,0.335 19,mlp.experts.55.up_proj,0.0001309230,0.01000,0.334 19,mlp.experts.56.up_proj,0.0006425231,0.01000,0.334 19,mlp.experts.57.up_proj,0.0001496243,0.01000,0.334 19,mlp.experts.58.up_proj,0.0001125335,0.01000,0.335 19,mlp.experts.59.up_proj,0.0000944897,0.01000,0.334 19,mlp.experts.60.up_proj,0.0004999095,0.01000,0.334 19,mlp.experts.61.up_proj,0.0001233148,0.01000,0.333 19,mlp.experts.62.up_proj,0.0000956986,0.01000,0.332 19,mlp.experts.63.up_proj,0.0000885485,0.01000,0.334 19,mlp.experts.0.down_proj,0.0000516270,0.01000,0.243 19,mlp.experts.1.down_proj,0.0000443423,0.01000,0.230 19,mlp.experts.2.down_proj,0.0000541653,0.01000,0.230 19,mlp.experts.3.down_proj,0.0000637528,0.01000,0.230 19,mlp.experts.4.down_proj,0.0000783231,0.01000,0.230 19,mlp.experts.5.down_proj,0.0000231497,0.01000,0.231 19,mlp.experts.6.down_proj,0.0000222858,0.01000,0.229 19,mlp.experts.7.down_proj,0.0004103894,0.01000,0.229 19,mlp.experts.8.down_proj,0.0000419891,0.01000,0.230 19,mlp.experts.9.down_proj,0.0003409567,0.01000,0.230 19,mlp.experts.10.down_proj,0.0000221717,0.01000,0.232 19,mlp.experts.11.down_proj,0.0000343662,0.01000,0.231 19,mlp.experts.12.down_proj,0.0000463175,0.01000,0.230 19,mlp.experts.13.down_proj,0.0001918597,0.01000,0.231 19,mlp.experts.14.down_proj,0.0000218438,0.01000,0.230 19,mlp.experts.15.down_proj,0.0000571246,0.01000,0.230 19,mlp.experts.16.down_proj,0.0000501294,0.01000,0.231 19,mlp.experts.17.down_proj,0.0000319578,0.01000,0.229 19,mlp.experts.18.down_proj,0.0000510123,0.01000,0.231 19,mlp.experts.19.down_proj,0.0005098363,0.01000,0.231 19,mlp.experts.20.down_proj,0.0000775645,0.01000,0.233 19,mlp.experts.21.down_proj,0.0000702166,0.01000,0.232 19,mlp.experts.22.down_proj,0.0000843812,0.01000,0.230 19,mlp.experts.23.down_proj,0.0000555512,0.01000,0.231 19,mlp.experts.24.down_proj,0.0004839645,0.01000,0.231 19,mlp.experts.25.down_proj,0.0000410045,0.01000,0.230 19,mlp.experts.26.down_proj,0.0000238551,0.01000,0.229 19,mlp.experts.27.down_proj,0.0000216650,0.01000,0.230 19,mlp.experts.28.down_proj,0.0000481833,0.01000,0.230 19,mlp.experts.29.down_proj,0.0000222296,0.01000,0.230 19,mlp.experts.30.down_proj,0.0000949479,0.01000,0.230 19,mlp.experts.31.down_proj,0.0000778862,0.01000,0.230 19,mlp.experts.32.down_proj,0.0000388138,0.01000,0.232 19,mlp.experts.33.down_proj,0.0001934957,0.01000,0.229 19,mlp.experts.34.down_proj,0.0000903839,0.01000,0.234 19,mlp.experts.35.down_proj,0.0000214777,0.01000,0.230 19,mlp.experts.36.down_proj,0.0004948681,0.01000,0.230 19,mlp.experts.37.down_proj,0.0000572881,0.01000,0.231 19,mlp.experts.38.down_proj,0.0001934959,0.01000,0.230 19,mlp.experts.39.down_proj,0.0000124011,0.01000,0.230 19,mlp.experts.40.down_proj,0.0000486178,0.01000,0.230 19,mlp.experts.41.down_proj,0.0000274658,0.01000,0.230 19,mlp.experts.42.down_proj,0.0000454438,0.01000,0.230 19,mlp.experts.43.down_proj,0.0000288570,0.01000,0.230 19,mlp.experts.44.down_proj,0.0000643459,0.01000,0.230 19,mlp.experts.45.down_proj,0.0000507622,0.01000,0.231 19,mlp.experts.46.down_proj,0.0000488614,0.01000,0.232 19,mlp.experts.47.down_proj,0.0000385272,0.01000,0.231 19,mlp.experts.48.down_proj,0.0000181821,0.01000,0.230 19,mlp.experts.49.down_proj,0.0045459368,0.01000,0.232 19,mlp.experts.50.down_proj,0.0000206987,0.01000,0.230 19,mlp.experts.51.down_proj,0.0000704047,0.01000,0.229 19,mlp.experts.52.down_proj,0.0000478366,0.01000,0.230 19,mlp.experts.53.down_proj,0.0000833012,0.01000,0.231 19,mlp.experts.54.down_proj,0.0000154871,0.01000,0.230 19,mlp.experts.55.down_proj,0.0000328368,0.01000,0.231 19,mlp.experts.56.down_proj,0.0002233989,0.01000,0.230 19,mlp.experts.57.down_proj,0.0000349312,0.01000,0.231 19,mlp.experts.58.down_proj,0.0000313825,0.01000,0.233 19,mlp.experts.59.down_proj,0.0000268332,0.01000,0.230 19,mlp.experts.60.down_proj,0.0001229730,0.01000,0.231 19,mlp.experts.61.down_proj,0.0000326987,0.01000,0.231 19,mlp.experts.62.down_proj,0.0000256997,0.01000,0.231 19,mlp.experts.63.down_proj,0.0000260199,0.01000,0.232 19,mlp.shared_experts.gate_proj,0.0000234016,0.01000,0.345 19,mlp.shared_experts.up_proj,0.0000206922,0.01000,0.331 19,mlp.shared_experts.down_proj,0.0000016866,0.01000,0.472 20,self_attn.kv_a_proj_with_mqa,0.0000059851,0.01000,0.348 20,self_attn.kv_b_proj,0.0000976203,0.01000,0.097 20,self_attn.o_proj,0.0000029646,0.01000,0.347 20,mlp.experts.0.gate_proj,0.0001910702,0.01000,0.347 20,mlp.experts.1.gate_proj,0.0003050595,0.01000,0.339 20,mlp.experts.2.gate_proj,0.0000723396,0.01000,0.333 20,mlp.experts.3.gate_proj,0.0006440120,0.01000,0.333 20,mlp.experts.4.gate_proj,0.0006052442,0.01000,0.334 20,mlp.experts.5.gate_proj,0.0000818279,0.01000,0.335 20,mlp.experts.6.gate_proj,0.0003806031,0.01000,0.336 20,mlp.experts.7.gate_proj,0.0001809356,0.01000,0.336 20,mlp.experts.8.gate_proj,0.0011225421,0.01000,0.334 20,mlp.experts.9.gate_proj,0.0001517877,0.01000,0.334 20,mlp.experts.10.gate_proj,0.0009826109,0.01000,0.336 20,mlp.experts.11.gate_proj,0.0002298986,0.01000,0.337 20,mlp.experts.12.gate_proj,0.0018987148,0.01000,0.335 20,mlp.experts.13.gate_proj,0.0002456404,0.01000,0.337 20,mlp.experts.14.gate_proj,0.0002100183,0.01000,0.336 20,mlp.experts.15.gate_proj,0.0002664563,0.01000,0.336 20,mlp.experts.16.gate_proj,0.0007221964,0.01000,0.335 20,mlp.experts.17.gate_proj,0.0008278585,0.01000,0.338 20,mlp.experts.18.gate_proj,0.0000469296,0.01000,0.336 20,mlp.experts.19.gate_proj,0.0004435812,0.01000,0.336 20,mlp.experts.20.gate_proj,0.0002448357,0.01000,0.336 20,mlp.experts.21.gate_proj,0.0001586104,0.01000,0.335 20,mlp.experts.22.gate_proj,0.0003198786,0.01000,0.334 20,mlp.experts.23.gate_proj,0.0005378306,0.01000,0.334 20,mlp.experts.24.gate_proj,0.0002494630,0.01000,0.337 20,mlp.experts.25.gate_proj,0.0000776144,0.01000,0.337 20,mlp.experts.26.gate_proj,0.0005494470,0.01000,0.336 20,mlp.experts.27.gate_proj,0.0002703760,0.01000,0.334 20,mlp.experts.28.gate_proj,0.0001409363,0.01000,0.336 20,mlp.experts.29.gate_proj,0.0018119293,0.01000,0.334 20,mlp.experts.30.gate_proj,0.0006828873,0.01000,0.334 20,mlp.experts.31.gate_proj,0.0003186909,0.01000,0.335 20,mlp.experts.32.gate_proj,0.0001822924,0.01000,0.336 20,mlp.experts.33.gate_proj,0.0000491391,0.01000,0.336 20,mlp.experts.34.gate_proj,0.0001782704,0.01000,0.335 20,mlp.experts.35.gate_proj,0.0001658642,0.01000,0.335 20,mlp.experts.36.gate_proj,0.0004923232,0.01000,0.336 20,mlp.experts.37.gate_proj,0.0002588543,0.01000,0.336 20,mlp.experts.38.gate_proj,0.0002075658,0.01000,0.334 20,mlp.experts.39.gate_proj,0.0005867743,0.01000,0.335 20,mlp.experts.40.gate_proj,0.0000835577,0.01000,0.335 20,mlp.experts.41.gate_proj,0.0001091276,0.01000,0.335 20,mlp.experts.42.gate_proj,0.0001456714,0.01000,0.334 20,mlp.experts.43.gate_proj,0.0005202589,0.01000,0.336 20,mlp.experts.44.gate_proj,0.0001901550,0.01000,0.336 20,mlp.experts.45.gate_proj,0.0001505947,0.01000,0.335 20,mlp.experts.46.gate_proj,0.0002649680,0.01000,0.334 20,mlp.experts.47.gate_proj,0.0002051032,0.01000,0.334 20,mlp.experts.48.gate_proj,0.0007578404,0.01000,0.336 20,mlp.experts.49.gate_proj,0.0005503986,0.01000,0.341 20,mlp.experts.50.gate_proj,0.0002153036,0.01000,0.335 20,mlp.experts.51.gate_proj,0.0003021561,0.01000,0.336 20,mlp.experts.52.gate_proj,0.0001021108,0.01000,0.337 20,mlp.experts.53.gate_proj,0.0034926427,0.01000,0.349 20,mlp.experts.54.gate_proj,0.0006917639,0.01000,0.335 20,mlp.experts.55.gate_proj,0.0000797021,0.01000,0.334 20,mlp.experts.56.gate_proj,0.0004956753,0.01000,0.335 20,mlp.experts.57.gate_proj,0.0002895309,0.01000,0.336 20,mlp.experts.58.gate_proj,0.0000946216,0.01000,0.335 20,mlp.experts.59.gate_proj,0.0015617721,0.01000,0.334 20,mlp.experts.60.gate_proj,0.0033675598,0.01000,0.335 20,mlp.experts.61.gate_proj,0.0005570114,0.01000,0.334 20,mlp.experts.62.gate_proj,0.0003657242,0.01000,0.335 20,mlp.experts.63.gate_proj,0.0001306589,0.01000,0.335 20,mlp.experts.0.up_proj,0.0001670428,0.01000,0.348 20,mlp.experts.1.up_proj,0.0002736492,0.01000,0.334 20,mlp.experts.2.up_proj,0.0000621941,0.01000,0.338 20,mlp.experts.3.up_proj,0.0005870386,0.01000,0.337 20,mlp.experts.4.up_proj,0.0005256841,0.01000,0.336 20,mlp.experts.5.up_proj,0.0000729494,0.01000,0.338 20,mlp.experts.6.up_proj,0.0003500764,0.01000,0.338 20,mlp.experts.7.up_proj,0.0001725662,0.01000,0.339 20,mlp.experts.8.up_proj,0.0010308883,0.01000,0.340 20,mlp.experts.9.up_proj,0.0001225731,0.01000,0.338 20,mlp.experts.10.up_proj,0.0008926830,0.01000,0.335 20,mlp.experts.11.up_proj,0.0002040629,0.01000,0.336 20,mlp.experts.12.up_proj,0.0016063645,0.01000,0.337 20,mlp.experts.13.up_proj,0.0002220221,0.01000,0.339 20,mlp.experts.14.up_proj,0.0001911036,0.01000,0.339 20,mlp.experts.15.up_proj,0.0002102089,0.01000,0.338 20,mlp.experts.16.up_proj,0.0006482315,0.01000,0.337 20,mlp.experts.17.up_proj,0.0007525291,0.01000,0.338 20,mlp.experts.18.up_proj,0.0000414573,0.01000,0.337 20,mlp.experts.19.up_proj,0.0004036963,0.01000,0.335 20,mlp.experts.20.up_proj,0.0002228606,0.01000,0.339 20,mlp.experts.21.up_proj,0.0001467516,0.01000,0.339 20,mlp.experts.22.up_proj,0.0002864778,0.01000,0.341 20,mlp.experts.23.up_proj,0.0004478686,0.01000,0.340 20,mlp.experts.24.up_proj,0.0002093227,0.01000,0.339 20,mlp.experts.25.up_proj,0.0000685404,0.01000,0.337 20,mlp.experts.26.up_proj,0.0005015306,0.01000,0.337 20,mlp.experts.27.up_proj,0.0002444570,0.01000,0.336 20,mlp.experts.28.up_proj,0.0001326581,0.01000,0.338 20,mlp.experts.29.up_proj,0.0016057962,0.01000,0.337 20,mlp.experts.30.up_proj,0.0006449441,0.01000,0.338 20,mlp.experts.31.up_proj,0.0002948477,0.01000,0.338 20,mlp.experts.32.up_proj,0.0001564313,0.01000,0.341 20,mlp.experts.33.up_proj,0.0000435593,0.01000,0.335 20,mlp.experts.34.up_proj,0.0001700695,0.01000,0.334 20,mlp.experts.35.up_proj,0.0001539697,0.01000,0.337 20,mlp.experts.36.up_proj,0.0004361027,0.01000,0.338 20,mlp.experts.37.up_proj,0.0002402903,0.01000,0.335 20,mlp.experts.38.up_proj,0.0001897946,0.01000,0.335 20,mlp.experts.39.up_proj,0.0004923930,0.01000,0.339 20,mlp.experts.40.up_proj,0.0000737153,0.01000,0.336 20,mlp.experts.41.up_proj,0.0001024155,0.01000,0.333 20,mlp.experts.42.up_proj,0.0001374328,0.01000,0.337 20,mlp.experts.43.up_proj,0.0004529061,0.01000,0.340 20,mlp.experts.44.up_proj,0.0001705071,0.01000,0.341 20,mlp.experts.45.up_proj,0.0001279940,0.01000,0.340 20,mlp.experts.46.up_proj,0.0002425315,0.01000,0.344 20,mlp.experts.47.up_proj,0.0001861247,0.01000,0.341 20,mlp.experts.48.up_proj,0.0006927405,0.01000,0.342 20,mlp.experts.49.up_proj,0.0005093247,0.01000,0.340 20,mlp.experts.50.up_proj,0.0001990946,0.01000,0.341 20,mlp.experts.51.up_proj,0.0002645554,0.01000,0.335 20,mlp.experts.52.up_proj,0.0000867168,0.01000,0.335 20,mlp.experts.53.up_proj,0.0031500595,0.01000,0.334 20,mlp.experts.54.up_proj,0.0006519017,0.01000,0.335 20,mlp.experts.55.up_proj,0.0000719762,0.01000,0.335 20,mlp.experts.56.up_proj,0.0004380725,0.01000,0.347 20,mlp.experts.57.up_proj,0.0002487250,0.01000,0.339 20,mlp.experts.58.up_proj,0.0000874907,0.01000,0.337 20,mlp.experts.59.up_proj,0.0014103951,0.01000,0.335 20,mlp.experts.60.up_proj,0.0029389307,0.01000,0.334 20,mlp.experts.61.up_proj,0.0005082715,0.01000,0.335 20,mlp.experts.62.up_proj,0.0003338464,0.01000,0.334 20,mlp.experts.63.up_proj,0.0001188974,0.01000,0.335 20,mlp.experts.0.down_proj,0.0000789434,0.01000,0.239 20,mlp.experts.1.down_proj,0.0000754304,0.01000,0.230 20,mlp.experts.2.down_proj,0.0000179413,0.01000,0.230 20,mlp.experts.3.down_proj,0.0001883605,0.01000,0.230 20,mlp.experts.4.down_proj,0.0001265344,0.01000,0.230 20,mlp.experts.5.down_proj,0.0000228667,0.01000,0.230 20,mlp.experts.6.down_proj,0.0001158572,0.01000,0.230 20,mlp.experts.7.down_proj,0.0000702539,0.01000,0.230 20,mlp.experts.8.down_proj,0.0002872758,0.01000,0.229 20,mlp.experts.9.down_proj,0.0000455196,0.01000,0.230 20,mlp.experts.10.down_proj,0.0002495587,0.01000,0.231 20,mlp.experts.11.down_proj,0.0000580857,0.01000,0.231 20,mlp.experts.12.down_proj,0.0004995728,0.01000,0.234 20,mlp.experts.13.down_proj,0.0000532547,0.01000,0.231 20,mlp.experts.14.down_proj,0.0000595258,0.01000,0.231 20,mlp.experts.15.down_proj,0.0000796263,0.01000,0.232 20,mlp.experts.16.down_proj,0.0002362328,0.01000,0.230 20,mlp.experts.17.down_proj,0.0002463557,0.01000,0.231 20,mlp.experts.18.down_proj,0.0000125020,0.01000,0.230 20,mlp.experts.19.down_proj,0.0001016197,0.01000,0.229 20,mlp.experts.20.down_proj,0.0000615250,0.01000,0.229 20,mlp.experts.21.down_proj,0.0000374060,0.01000,0.230 20,mlp.experts.22.down_proj,0.0000691374,0.01000,0.229 20,mlp.experts.23.down_proj,0.0001386564,0.01000,0.230 20,mlp.experts.24.down_proj,0.0000486573,0.01000,0.231 20,mlp.experts.25.down_proj,0.0000264864,0.01000,0.231 20,mlp.experts.26.down_proj,0.0001456150,0.01000,0.230 20,mlp.experts.27.down_proj,0.0000636890,0.01000,0.230 20,mlp.experts.28.down_proj,0.0000416002,0.01000,0.229 20,mlp.experts.29.down_proj,0.0004730057,0.01000,0.230 20,mlp.experts.30.down_proj,0.0001765976,0.01000,0.229 20,mlp.experts.31.down_proj,0.0000987379,0.01000,0.228 20,mlp.experts.32.down_proj,0.0000419839,0.01000,0.229 20,mlp.experts.33.down_proj,0.0000112639,0.01000,0.230 20,mlp.experts.34.down_proj,0.0000525765,0.01000,0.230 20,mlp.experts.35.down_proj,0.0000448138,0.01000,0.230 20,mlp.experts.36.down_proj,0.0001539926,0.01000,0.230 20,mlp.experts.37.down_proj,0.0000589089,0.01000,0.230 20,mlp.experts.38.down_proj,0.0000513632,0.01000,0.231 20,mlp.experts.39.down_proj,0.0001362028,0.01000,0.230 20,mlp.experts.40.down_proj,0.0000226174,0.01000,0.230 20,mlp.experts.41.down_proj,0.0000251516,0.01000,0.231 20,mlp.experts.42.down_proj,0.0000343938,0.01000,0.231 20,mlp.experts.43.down_proj,0.0001282972,0.01000,0.230 20,mlp.experts.44.down_proj,0.0000474056,0.01000,0.231 20,mlp.experts.45.down_proj,0.0000374316,0.01000,0.230 20,mlp.experts.46.down_proj,0.0000565848,0.01000,0.231 20,mlp.experts.47.down_proj,0.0000613344,0.01000,0.231 20,mlp.experts.48.down_proj,0.0002038087,0.01000,0.229 20,mlp.experts.49.down_proj,0.0001517802,0.01000,0.231 20,mlp.experts.50.down_proj,0.0000594156,0.01000,0.231 20,mlp.experts.51.down_proj,0.0000886968,0.01000,0.230 20,mlp.experts.52.down_proj,0.0000231733,0.01000,0.230 20,mlp.experts.53.down_proj,0.0010732206,0.01000,0.230 20,mlp.experts.54.down_proj,0.0002007890,0.01000,0.230 20,mlp.experts.55.down_proj,0.0000179043,0.01000,0.230 20,mlp.experts.56.down_proj,0.0001793337,0.01000,0.231 20,mlp.experts.57.down_proj,0.0000631149,0.01000,0.230 20,mlp.experts.58.down_proj,0.0000249460,0.01000,0.231 20,mlp.experts.59.down_proj,0.0004696032,0.01000,0.229 20,mlp.experts.60.down_proj,0.0009232018,0.01000,0.229 20,mlp.experts.61.down_proj,0.0001414192,0.01000,0.230 20,mlp.experts.62.down_proj,0.0000840021,0.01000,0.230 20,mlp.experts.63.down_proj,0.0000317826,0.01000,0.231 20,mlp.shared_experts.gate_proj,0.0000242172,0.01000,0.344 20,mlp.shared_experts.up_proj,0.0000201628,0.01000,0.332 20,mlp.shared_experts.down_proj,0.0000017881,0.01000,0.473 21,self_attn.kv_a_proj_with_mqa,0.0000059686,0.01000,0.350 21,self_attn.kv_b_proj,0.0002334213,0.01000,0.094 21,self_attn.o_proj,0.0000119160,0.01000,0.343 21,mlp.experts.0.gate_proj,0.0001830408,0.01000,0.347 21,mlp.experts.1.gate_proj,0.0119938998,0.01000,0.337 21,mlp.experts.2.gate_proj,0.0001520745,0.01000,0.336 21,mlp.experts.3.gate_proj,0.0001463988,0.01000,0.337 21,mlp.experts.4.gate_proj,0.0024747673,0.01000,0.339 21,mlp.experts.5.gate_proj,0.0002842203,0.01000,0.340 21,mlp.experts.6.gate_proj,0.0001223941,0.01000,0.340 21,mlp.experts.7.gate_proj,0.0001110525,0.01000,0.337 21,mlp.experts.8.gate_proj,0.0001247081,0.01000,0.339 21,mlp.experts.9.gate_proj,0.0002962266,0.01000,0.343 21,mlp.experts.10.gate_proj,0.0005602464,0.01000,0.339 21,mlp.experts.11.gate_proj,0.0003037366,0.01000,0.339 21,mlp.experts.12.gate_proj,0.0003140313,0.01000,0.337 21,mlp.experts.13.gate_proj,0.0002403335,0.01000,0.336 21,mlp.experts.14.gate_proj,0.0001278773,0.01000,0.335 21,mlp.experts.15.gate_proj,0.0002940982,0.01000,0.335 21,mlp.experts.16.gate_proj,0.0001227750,0.01000,0.335 21,mlp.experts.17.gate_proj,0.0002294234,0.01000,0.336 21,mlp.experts.18.gate_proj,0.0006960069,0.01000,0.336 21,mlp.experts.19.gate_proj,0.0000979176,0.01000,0.335 21,mlp.experts.20.gate_proj,0.0015217504,0.01000,0.336 21,mlp.experts.21.gate_proj,0.0002657666,0.01000,0.335 21,mlp.experts.22.gate_proj,0.0004300820,0.01000,0.336 21,mlp.experts.23.gate_proj,0.0071323497,0.01000,0.337 21,mlp.experts.24.gate_proj,0.0001065242,0.01000,0.337 21,mlp.experts.25.gate_proj,0.0001281302,0.01000,0.337 21,mlp.experts.26.gate_proj,0.0003270586,0.01000,0.336 21,mlp.experts.27.gate_proj,0.0004336565,0.01000,0.338 21,mlp.experts.28.gate_proj,0.0002429369,0.01000,0.338 21,mlp.experts.29.gate_proj,0.0011122219,0.01000,0.336 21,mlp.experts.30.gate_proj,0.0011935835,0.01000,0.336 21,mlp.experts.31.gate_proj,0.0001140919,0.01000,0.337 21,mlp.experts.32.gate_proj,0.0014129694,0.01000,0.335 21,mlp.experts.33.gate_proj,0.0006096888,0.01000,0.339 21,mlp.experts.34.gate_proj,0.0004015109,0.01000,0.335 21,mlp.experts.35.gate_proj,0.0001227568,0.01000,0.336 21,mlp.experts.36.gate_proj,0.0002604676,0.01000,0.334 21,mlp.experts.37.gate_proj,0.0006790702,0.01000,0.337 21,mlp.experts.38.gate_proj,0.0001804348,0.01000,0.336 21,mlp.experts.39.gate_proj,0.0002531391,0.01000,0.339 21,mlp.experts.40.gate_proj,0.0003119032,0.01000,0.351 21,mlp.experts.41.gate_proj,0.0002390300,0.01000,0.336 21,mlp.experts.42.gate_proj,0.0002517759,0.01000,0.334 21,mlp.experts.43.gate_proj,0.0003253688,0.01000,0.336 21,mlp.experts.44.gate_proj,0.0018552641,0.01000,0.335 21,mlp.experts.45.gate_proj,0.0001736703,0.01000,0.337 21,mlp.experts.46.gate_proj,0.0002443869,0.01000,0.336 21,mlp.experts.47.gate_proj,0.0003024441,0.01000,0.336 21,mlp.experts.48.gate_proj,0.0002358153,0.01000,0.336 21,mlp.experts.49.gate_proj,0.0000752732,0.01000,0.336 21,mlp.experts.50.gate_proj,0.0003386062,0.01000,0.335 21,mlp.experts.51.gate_proj,0.0005414677,0.01000,0.335 21,mlp.experts.52.gate_proj,0.0000749865,0.01000,0.334 21,mlp.experts.53.gate_proj,0.0001198875,0.01000,0.337 21,mlp.experts.54.gate_proj,0.0003014246,0.01000,0.336 21,mlp.experts.55.gate_proj,0.0014737491,0.01000,0.335 21,mlp.experts.56.gate_proj,0.0001826291,0.01000,0.334 21,mlp.experts.57.gate_proj,0.0003603130,0.01000,0.337 21,mlp.experts.58.gate_proj,0.0005495258,0.01000,0.337 21,mlp.experts.59.gate_proj,0.0001906526,0.01000,0.334 21,mlp.experts.60.gate_proj,0.0009344295,0.01000,0.335 21,mlp.experts.61.gate_proj,0.0004157704,0.01000,0.335 21,mlp.experts.62.gate_proj,0.0001898678,0.01000,0.335 21,mlp.experts.63.gate_proj,0.0004739862,0.01000,0.334 21,mlp.experts.0.up_proj,0.0001660666,0.01000,0.349 21,mlp.experts.1.up_proj,0.0099509271,0.01000,0.337 21,mlp.experts.2.up_proj,0.0001403952,0.01000,0.335 21,mlp.experts.3.up_proj,0.0001367296,0.01000,0.335 21,mlp.experts.4.up_proj,0.0019553380,0.01000,0.335 21,mlp.experts.5.up_proj,0.0002649712,0.01000,0.336 21,mlp.experts.6.up_proj,0.0001122793,0.01000,0.339 21,mlp.experts.7.up_proj,0.0001007799,0.01000,0.337 21,mlp.experts.8.up_proj,0.0001109353,0.01000,0.339 21,mlp.experts.9.up_proj,0.0002683810,0.01000,0.339 21,mlp.experts.10.up_proj,0.0005246236,0.01000,0.338 21,mlp.experts.11.up_proj,0.0002760288,0.01000,0.335 21,mlp.experts.12.up_proj,0.0002915841,0.01000,0.335 21,mlp.experts.13.up_proj,0.0002243770,0.01000,0.335 21,mlp.experts.14.up_proj,0.0001180603,0.01000,0.335 21,mlp.experts.15.up_proj,0.0002646645,0.01000,0.337 21,mlp.experts.16.up_proj,0.0001135548,0.01000,0.336 21,mlp.experts.17.up_proj,0.0002093067,0.01000,0.337 21,mlp.experts.18.up_proj,0.0006134511,0.01000,0.337 21,mlp.experts.19.up_proj,0.0000888784,0.01000,0.337 21,mlp.experts.20.up_proj,0.0013397215,0.01000,0.335 21,mlp.experts.21.up_proj,0.0002422696,0.01000,0.337 21,mlp.experts.22.up_proj,0.0003931007,0.01000,0.335 21,mlp.experts.23.up_proj,0.0063861325,0.01000,0.336 21,mlp.experts.24.up_proj,0.0000986955,0.01000,0.334 21,mlp.experts.25.up_proj,0.0001150009,0.01000,0.336 21,mlp.experts.26.up_proj,0.0002946560,0.01000,0.337 21,mlp.experts.27.up_proj,0.0004029040,0.01000,0.337 21,mlp.experts.28.up_proj,0.0002241887,0.01000,0.337 21,mlp.experts.29.up_proj,0.0010492990,0.01000,0.337 21,mlp.experts.30.up_proj,0.0010994067,0.01000,0.336 21,mlp.experts.31.up_proj,0.0001051230,0.01000,0.337 21,mlp.experts.32.up_proj,0.0013187394,0.01000,0.336 21,mlp.experts.33.up_proj,0.0005575601,0.01000,0.336 21,mlp.experts.34.up_proj,0.0003751171,0.01000,0.339 21,mlp.experts.35.up_proj,0.0001169315,0.01000,0.339 21,mlp.experts.36.up_proj,0.0002440056,0.01000,0.335 21,mlp.experts.37.up_proj,0.0005838174,0.01000,0.337 21,mlp.experts.38.up_proj,0.0001618690,0.01000,0.336 21,mlp.experts.39.up_proj,0.0002297707,0.01000,0.335 21,mlp.experts.40.up_proj,0.0002771356,0.01000,0.336 21,mlp.experts.41.up_proj,0.0002007657,0.01000,0.336 21,mlp.experts.42.up_proj,0.0002334783,0.01000,0.338 21,mlp.experts.43.up_proj,0.0003015150,0.01000,0.336 21,mlp.experts.44.up_proj,0.0015469883,0.01000,0.335 21,mlp.experts.45.up_proj,0.0001676577,0.01000,0.335 21,mlp.experts.46.up_proj,0.0002226182,0.01000,0.336 21,mlp.experts.47.up_proj,0.0002650861,0.01000,0.336 21,mlp.experts.48.up_proj,0.0002240956,0.01000,0.340 21,mlp.experts.49.up_proj,0.0000668052,0.01000,0.339 21,mlp.experts.50.up_proj,0.0003157143,0.01000,0.339 21,mlp.experts.51.up_proj,0.0005062145,0.01000,0.340 21,mlp.experts.52.up_proj,0.0000678234,0.01000,0.341 21,mlp.experts.53.up_proj,0.0001127448,0.01000,0.338 21,mlp.experts.54.up_proj,0.0002628491,0.01000,0.336 21,mlp.experts.55.up_proj,0.0013373234,0.01000,0.336 21,mlp.experts.56.up_proj,0.0001702486,0.01000,0.337 21,mlp.experts.57.up_proj,0.0003417460,0.01000,0.336 21,mlp.experts.58.up_proj,0.0005122946,0.01000,0.337 21,mlp.experts.59.up_proj,0.0001778237,0.01000,0.339 21,mlp.experts.60.up_proj,0.0008848563,0.01000,0.337 21,mlp.experts.61.up_proj,0.0003816825,0.01000,0.336 21,mlp.experts.62.up_proj,0.0001666123,0.01000,0.334 21,mlp.experts.63.up_proj,0.0004459308,0.01000,0.336 21,mlp.experts.0.down_proj,0.0000604285,0.01000,0.242 21,mlp.experts.1.down_proj,0.0035495605,0.01000,0.231 21,mlp.experts.2.down_proj,0.0000375245,0.01000,0.234 21,mlp.experts.3.down_proj,0.0000499589,0.01000,0.232 21,mlp.experts.4.down_proj,0.0008717987,0.01000,0.233 21,mlp.experts.5.down_proj,0.0000881609,0.01000,0.233 21,mlp.experts.6.down_proj,0.0000428355,0.01000,0.233 21,mlp.experts.7.down_proj,0.0000403577,0.01000,0.232 21,mlp.experts.8.down_proj,0.0000307811,0.01000,0.233 21,mlp.experts.9.down_proj,0.0000992082,0.01000,0.234 21,mlp.experts.10.down_proj,0.0001928519,0.01000,0.232 21,mlp.experts.11.down_proj,0.0000922271,0.01000,0.230 21,mlp.experts.12.down_proj,0.0000875301,0.01000,0.232 21,mlp.experts.13.down_proj,0.0000682987,0.01000,0.230 21,mlp.experts.14.down_proj,0.0000358917,0.01000,0.233 21,mlp.experts.15.down_proj,0.0000913298,0.01000,0.231 21,mlp.experts.16.down_proj,0.0000441483,0.01000,0.231 21,mlp.experts.17.down_proj,0.0000656080,0.01000,0.231 21,mlp.experts.18.down_proj,0.0002337122,0.01000,0.231 21,mlp.experts.19.down_proj,0.0000305546,0.01000,0.235 21,mlp.experts.20.down_proj,0.0005653448,0.01000,0.231 21,mlp.experts.21.down_proj,0.0000780814,0.01000,0.232 21,mlp.experts.22.down_proj,0.0001467234,0.01000,0.231 21,mlp.experts.23.down_proj,0.0023442241,0.01000,0.230 21,mlp.experts.24.down_proj,0.0000312360,0.01000,0.234 21,mlp.experts.25.down_proj,0.0000421724,0.01000,0.236 21,mlp.experts.26.down_proj,0.0000880099,0.01000,0.229 21,mlp.experts.27.down_proj,0.0001227354,0.01000,0.231 21,mlp.experts.28.down_proj,0.0000717244,0.01000,0.231 21,mlp.experts.29.down_proj,0.0003526885,0.01000,0.231 21,mlp.experts.30.down_proj,0.0004267962,0.01000,0.229 21,mlp.experts.31.down_proj,0.0000309216,0.01000,0.229 21,mlp.experts.32.down_proj,0.0004467358,0.01000,0.230 21,mlp.experts.33.down_proj,0.0001785831,0.01000,0.229 21,mlp.experts.34.down_proj,0.0001350764,0.01000,0.229 21,mlp.experts.35.down_proj,0.0000459272,0.01000,0.230 21,mlp.experts.36.down_proj,0.0000744385,0.01000,0.231 21,mlp.experts.37.down_proj,0.0001820413,0.01000,0.230 21,mlp.experts.38.down_proj,0.0000646221,0.01000,0.230 21,mlp.experts.39.down_proj,0.0000754553,0.01000,0.230 21,mlp.experts.40.down_proj,0.0001145490,0.01000,0.231 21,mlp.experts.41.down_proj,0.0000635686,0.01000,0.230 21,mlp.experts.42.down_proj,0.0000740115,0.01000,0.230 21,mlp.experts.43.down_proj,0.0001038310,0.01000,0.230 21,mlp.experts.44.down_proj,0.0005995781,0.01000,0.230 21,mlp.experts.45.down_proj,0.0000600322,0.01000,0.230 21,mlp.experts.46.down_proj,0.0000912117,0.01000,0.231 21,mlp.experts.47.down_proj,0.0001024066,0.01000,0.229 21,mlp.experts.48.down_proj,0.0000994911,0.01000,0.232 21,mlp.experts.49.down_proj,0.0000196013,0.01000,0.230 21,mlp.experts.50.down_proj,0.0001096951,0.01000,0.229 21,mlp.experts.51.down_proj,0.0002404264,0.01000,0.230 21,mlp.experts.52.down_proj,0.0000225512,0.01000,0.230 21,mlp.experts.53.down_proj,0.0000328746,0.01000,0.229 21,mlp.experts.54.down_proj,0.0001024319,0.01000,0.229 21,mlp.experts.55.down_proj,0.0004458318,0.01000,0.230 21,mlp.experts.56.down_proj,0.0000628367,0.01000,0.229 21,mlp.experts.57.down_proj,0.0001099426,0.01000,0.230 21,mlp.experts.58.down_proj,0.0001826156,0.01000,0.229 21,mlp.experts.59.down_proj,0.0000584280,0.01000,0.230 21,mlp.experts.60.down_proj,0.0002981232,0.01000,0.230 21,mlp.experts.61.down_proj,0.0001215974,0.01000,0.231 21,mlp.experts.62.down_proj,0.0000526441,0.01000,0.230 21,mlp.experts.63.down_proj,0.0001440406,0.01000,0.229 21,mlp.shared_experts.gate_proj,0.0000288040,0.01000,0.344 21,mlp.shared_experts.up_proj,0.0000222900,0.01000,0.335 21,mlp.shared_experts.down_proj,0.0000017100,0.01000,0.469 22,self_attn.kv_a_proj_with_mqa,0.0000066915,0.01000,0.337 22,self_attn.kv_b_proj,0.0001944743,0.01000,0.098 22,self_attn.o_proj,0.0000070872,0.01000,0.344 22,mlp.experts.0.gate_proj,0.0003054985,0.01000,0.347 22,mlp.experts.1.gate_proj,0.0010742689,0.01000,0.338 22,mlp.experts.2.gate_proj,0.0002312400,0.01000,0.336 22,mlp.experts.3.gate_proj,0.0003575801,0.01000,0.337 22,mlp.experts.4.gate_proj,0.0017972376,0.01000,0.336 22,mlp.experts.5.gate_proj,0.0003218201,0.01000,0.337 22,mlp.experts.6.gate_proj,0.0008333748,0.01000,0.337 22,mlp.experts.7.gate_proj,0.0004194044,0.01000,0.336 22,mlp.experts.8.gate_proj,0.0009884146,0.01000,0.337 22,mlp.experts.9.gate_proj,0.0003689468,0.01000,0.337 22,mlp.experts.10.gate_proj,0.0014711401,0.01000,0.337 22,mlp.experts.11.gate_proj,0.0003187458,0.01000,0.336 22,mlp.experts.12.gate_proj,0.0006993395,0.01000,0.335 22,mlp.experts.13.gate_proj,0.0001229152,0.01000,0.335 22,mlp.experts.14.gate_proj,0.0001609310,0.01000,0.335 22,mlp.experts.15.gate_proj,0.0021269891,0.01000,0.337 22,mlp.experts.16.gate_proj,0.0214003384,0.01000,0.337 22,mlp.experts.17.gate_proj,0.0002286402,0.01000,0.338 22,mlp.experts.18.gate_proj,0.0004076681,0.01000,0.339 22,mlp.experts.19.gate_proj,0.0011497557,0.01000,0.337 22,mlp.experts.20.gate_proj,0.0001964248,0.01000,0.337 22,mlp.experts.21.gate_proj,0.0001032768,0.01000,0.337 22,mlp.experts.22.gate_proj,0.0001975987,0.01000,0.335 22,mlp.experts.23.gate_proj,0.0006940385,0.01000,0.336 22,mlp.experts.24.gate_proj,0.0001918491,0.01000,0.340 22,mlp.experts.25.gate_proj,0.0003068354,0.01000,0.337 22,mlp.experts.26.gate_proj,0.0009232846,0.01000,0.339 22,mlp.experts.27.gate_proj,0.0005465810,0.01000,0.341 22,mlp.experts.28.gate_proj,0.0002984151,0.01000,0.339 22,mlp.experts.29.gate_proj,0.0033758616,0.01000,0.336 22,mlp.experts.30.gate_proj,0.0001695116,0.01000,0.337 22,mlp.experts.31.gate_proj,0.0002208285,0.01000,0.335 22,mlp.experts.32.gate_proj,0.0003459042,0.01000,0.336 22,mlp.experts.33.gate_proj,0.0003791910,0.01000,0.337 22,mlp.experts.34.gate_proj,0.0004034041,0.01000,0.338 22,mlp.experts.35.gate_proj,0.0000744882,0.01000,0.337 22,mlp.experts.36.gate_proj,0.0001080414,0.01000,0.337 22,mlp.experts.37.gate_proj,0.0008720923,0.01000,0.337 22,mlp.experts.38.gate_proj,0.0002456164,0.01000,0.338 22,mlp.experts.39.gate_proj,0.0003546432,0.01000,0.339 22,mlp.experts.40.gate_proj,0.0005031173,0.01000,0.340 22,mlp.experts.41.gate_proj,0.0001809030,0.01000,0.341 22,mlp.experts.42.gate_proj,0.0005331776,0.01000,0.336 22,mlp.experts.43.gate_proj,0.0001093682,0.01000,0.341 22,mlp.experts.44.gate_proj,0.0002378691,0.01000,0.337 22,mlp.experts.45.gate_proj,0.0006978381,0.01000,0.336 22,mlp.experts.46.gate_proj,0.0000854732,0.01000,0.335 22,mlp.experts.47.gate_proj,0.0006663075,0.01000,0.333 22,mlp.experts.48.gate_proj,0.0003399646,0.01000,0.335 22,mlp.experts.49.gate_proj,0.0001967158,0.01000,0.333 22,mlp.experts.50.gate_proj,0.0000845789,0.01000,0.334 22,mlp.experts.51.gate_proj,0.0001714878,0.01000,0.336 22,mlp.experts.52.gate_proj,0.0015025136,0.01000,0.335 22,mlp.experts.53.gate_proj,0.0004102380,0.01000,0.335 22,mlp.experts.54.gate_proj,0.0003153585,0.01000,0.337 22,mlp.experts.55.gate_proj,0.0022654425,0.01000,0.336 22,mlp.experts.56.gate_proj,0.0002262286,0.01000,0.337 22,mlp.experts.57.gate_proj,0.0004517708,0.01000,0.335 22,mlp.experts.58.gate_proj,0.0008440733,0.01000,0.335 22,mlp.experts.59.gate_proj,0.0001310003,0.01000,0.335 22,mlp.experts.60.gate_proj,0.0009930636,0.01000,0.335 22,mlp.experts.61.gate_proj,0.0003082075,0.01000,0.336 22,mlp.experts.62.gate_proj,0.0002798371,0.01000,0.335 22,mlp.experts.63.gate_proj,0.0003471757,0.01000,0.334 22,mlp.experts.0.up_proj,0.0002885296,0.01000,0.347 22,mlp.experts.1.up_proj,0.0010536595,0.01000,0.335 22,mlp.experts.2.up_proj,0.0002205143,0.01000,0.334 22,mlp.experts.3.up_proj,0.0003415266,0.01000,0.335 22,mlp.experts.4.up_proj,0.0015693573,0.01000,0.336 22,mlp.experts.5.up_proj,0.0002989684,0.01000,0.337 22,mlp.experts.6.up_proj,0.0008075996,0.01000,0.336 22,mlp.experts.7.up_proj,0.0003849928,0.01000,0.336 22,mlp.experts.8.up_proj,0.0009373176,0.01000,0.336 22,mlp.experts.9.up_proj,0.0003484287,0.01000,0.338 22,mlp.experts.10.up_proj,0.0013591747,0.01000,0.336 22,mlp.experts.11.up_proj,0.0002964965,0.01000,0.337 22,mlp.experts.12.up_proj,0.0006129925,0.01000,0.337 22,mlp.experts.13.up_proj,0.0001119205,0.01000,0.336 22,mlp.experts.14.up_proj,0.0001498496,0.01000,0.336 22,mlp.experts.15.up_proj,0.0017142597,0.01000,0.333 22,mlp.experts.16.up_proj,0.0171640610,0.01000,0.335 22,mlp.experts.17.up_proj,0.0002117203,0.01000,0.336 22,mlp.experts.18.up_proj,0.0003908697,0.01000,0.338 22,mlp.experts.19.up_proj,0.0010825896,0.01000,0.336 22,mlp.experts.20.up_proj,0.0001652615,0.01000,0.334 22,mlp.experts.21.up_proj,0.0000871432,0.01000,0.334 22,mlp.experts.22.up_proj,0.0001770025,0.01000,0.335 22,mlp.experts.23.up_proj,0.0006634577,0.01000,0.334 22,mlp.experts.24.up_proj,0.0001664612,0.01000,0.335 22,mlp.experts.25.up_proj,0.0002740491,0.01000,0.336 22,mlp.experts.26.up_proj,0.0008751037,0.01000,0.336 22,mlp.experts.27.up_proj,0.0004680914,0.01000,0.347 22,mlp.experts.28.up_proj,0.0002680988,0.01000,0.336 22,mlp.experts.29.up_proj,0.0032674009,0.01000,0.334 22,mlp.experts.30.up_proj,0.0001503322,0.01000,0.334 22,mlp.experts.31.up_proj,0.0002097403,0.01000,0.334 22,mlp.experts.32.up_proj,0.0003202185,0.01000,0.334 22,mlp.experts.33.up_proj,0.0003588597,0.01000,0.335 22,mlp.experts.34.up_proj,0.0003812558,0.01000,0.334 22,mlp.experts.35.up_proj,0.0000703603,0.01000,0.334 22,mlp.experts.36.up_proj,0.0001006523,0.01000,0.335 22,mlp.experts.37.up_proj,0.0007971759,0.01000,0.335 22,mlp.experts.38.up_proj,0.0002094727,0.01000,0.334 22,mlp.experts.39.up_proj,0.0003378264,0.01000,0.336 22,mlp.experts.40.up_proj,0.0004665519,0.01000,0.335 22,mlp.experts.41.up_proj,0.0001645600,0.01000,0.334 22,mlp.experts.42.up_proj,0.0004635091,0.01000,0.335 22,mlp.experts.43.up_proj,0.0000945225,0.01000,0.335 22,mlp.experts.44.up_proj,0.0002208673,0.01000,0.334 22,mlp.experts.45.up_proj,0.0006583503,0.01000,0.334 22,mlp.experts.46.up_proj,0.0000807889,0.01000,0.333 22,mlp.experts.47.up_proj,0.0006193824,0.01000,0.336 22,mlp.experts.48.up_proj,0.0003173202,0.01000,0.335 22,mlp.experts.49.up_proj,0.0001817779,0.01000,0.336 22,mlp.experts.50.up_proj,0.0000756650,0.01000,0.336 22,mlp.experts.51.up_proj,0.0001546787,0.01000,0.336 22,mlp.experts.52.up_proj,0.0012310394,0.01000,0.337 22,mlp.experts.53.up_proj,0.0004044672,0.01000,0.336 22,mlp.experts.54.up_proj,0.0003091176,0.01000,0.337 22,mlp.experts.55.up_proj,0.0019459228,0.01000,0.336 22,mlp.experts.56.up_proj,0.0002082375,0.01000,0.336 22,mlp.experts.57.up_proj,0.0004168321,0.01000,0.335 22,mlp.experts.58.up_proj,0.0008078280,0.01000,0.337 22,mlp.experts.59.up_proj,0.0001180157,0.01000,0.337 22,mlp.experts.60.up_proj,0.0009557785,0.01000,0.336 22,mlp.experts.61.up_proj,0.0002819948,0.01000,0.335 22,mlp.experts.62.up_proj,0.0002662876,0.01000,0.336 22,mlp.experts.63.up_proj,0.0002909569,0.01000,0.337 22,mlp.experts.0.down_proj,0.0001040114,0.01000,0.243 22,mlp.experts.1.down_proj,0.0003880079,0.01000,0.229 22,mlp.experts.2.down_proj,0.0000857387,0.01000,0.230 22,mlp.experts.3.down_proj,0.0001284634,0.01000,0.231 22,mlp.experts.4.down_proj,0.0008655071,0.01000,0.230 22,mlp.experts.5.down_proj,0.0001179968,0.01000,0.230 22,mlp.experts.6.down_proj,0.0003474617,0.01000,0.230 22,mlp.experts.7.down_proj,0.0001332567,0.01000,0.230 22,mlp.experts.8.down_proj,0.0003996935,0.01000,0.231 22,mlp.experts.9.down_proj,0.0001406769,0.01000,0.229 22,mlp.experts.10.down_proj,0.0004887821,0.01000,0.231 22,mlp.experts.11.down_proj,0.0000919097,0.01000,0.230 22,mlp.experts.12.down_proj,0.0002384096,0.01000,0.231 22,mlp.experts.13.down_proj,0.0000384452,0.01000,0.229 22,mlp.experts.14.down_proj,0.0000510493,0.01000,0.230 22,mlp.experts.15.down_proj,0.0008581062,0.01000,0.231 22,mlp.experts.16.down_proj,0.0070998961,0.01000,0.229 22,mlp.experts.17.down_proj,0.0000786915,0.01000,0.231 22,mlp.experts.18.down_proj,0.0001750352,0.01000,0.230 22,mlp.experts.19.down_proj,0.0004241949,0.01000,0.231 22,mlp.experts.20.down_proj,0.0000841194,0.01000,0.230 22,mlp.experts.21.down_proj,0.0000409116,0.01000,0.229 22,mlp.experts.22.down_proj,0.0000850232,0.01000,0.243 22,mlp.experts.23.down_proj,0.0002368825,0.01000,0.230 22,mlp.experts.24.down_proj,0.0000662463,0.01000,0.229 22,mlp.experts.25.down_proj,0.0001176608,0.01000,0.233 22,mlp.experts.26.down_proj,0.0003197437,0.01000,0.229 22,mlp.experts.27.down_proj,0.0002438960,0.01000,0.230 22,mlp.experts.28.down_proj,0.0000863529,0.01000,0.230 22,mlp.experts.29.down_proj,0.0013967475,0.01000,0.230 22,mlp.experts.30.down_proj,0.0000631140,0.01000,0.230 22,mlp.experts.31.down_proj,0.0000836824,0.01000,0.229 22,mlp.experts.32.down_proj,0.0000993262,0.01000,0.231 22,mlp.experts.33.down_proj,0.0001206078,0.01000,0.231 22,mlp.experts.34.down_proj,0.0001929744,0.01000,0.231 22,mlp.experts.35.down_proj,0.0000251000,0.01000,0.230 22,mlp.experts.36.down_proj,0.0000416591,0.01000,0.230 22,mlp.experts.37.down_proj,0.0003263570,0.01000,0.230 22,mlp.experts.38.down_proj,0.0000993536,0.01000,0.230 22,mlp.experts.39.down_proj,0.0001501064,0.01000,0.230 22,mlp.experts.40.down_proj,0.0001567138,0.01000,0.230 22,mlp.experts.41.down_proj,0.0000666459,0.01000,0.230 22,mlp.experts.42.down_proj,0.0001976770,0.01000,0.229 22,mlp.experts.43.down_proj,0.0000366002,0.01000,0.231 22,mlp.experts.44.down_proj,0.0000877387,0.01000,0.231 22,mlp.experts.45.down_proj,0.0002606924,0.01000,0.230 22,mlp.experts.46.down_proj,0.0000356905,0.01000,0.229 22,mlp.experts.47.down_proj,0.0002554629,0.01000,0.231 22,mlp.experts.48.down_proj,0.0001140812,0.01000,0.232 22,mlp.experts.49.down_proj,0.0000651117,0.01000,0.231 22,mlp.experts.50.down_proj,0.0000346709,0.01000,0.232 22,mlp.experts.51.down_proj,0.0000715715,0.01000,0.232 22,mlp.experts.52.down_proj,0.0004226631,0.01000,0.231 22,mlp.experts.53.down_proj,0.0001703080,0.01000,0.233 22,mlp.experts.54.down_proj,0.0001097921,0.01000,0.231 22,mlp.experts.55.down_proj,0.0007336871,0.01000,0.231 22,mlp.experts.56.down_proj,0.0000827567,0.01000,0.231 22,mlp.experts.57.down_proj,0.0001363096,0.01000,0.232 22,mlp.experts.58.down_proj,0.0003173338,0.01000,0.231 22,mlp.experts.59.down_proj,0.0000456892,0.01000,0.231 22,mlp.experts.60.down_proj,0.0004003052,0.01000,0.232 22,mlp.experts.61.down_proj,0.0001191542,0.01000,0.230 22,mlp.experts.62.down_proj,0.0001029122,0.01000,0.231 22,mlp.experts.63.down_proj,0.0001118833,0.01000,0.230 22,mlp.shared_experts.gate_proj,0.0000312349,0.01000,0.346 22,mlp.shared_experts.up_proj,0.0000243878,0.01000,0.336 22,mlp.shared_experts.down_proj,0.0000022697,0.01000,0.474 23,self_attn.kv_a_proj_with_mqa,0.0000063790,0.01000,0.351 23,self_attn.kv_b_proj,0.0002630889,0.01000,0.096 23,self_attn.o_proj,0.0000218662,0.01000,0.343 23,mlp.experts.0.gate_proj,0.0003847479,0.01000,0.348 23,mlp.experts.1.gate_proj,0.0002364015,0.01000,0.336 23,mlp.experts.2.gate_proj,0.0001560770,0.01000,0.338 23,mlp.experts.3.gate_proj,0.0004788036,0.01000,0.335 23,mlp.experts.4.gate_proj,0.0001593112,0.01000,0.336 23,mlp.experts.5.gate_proj,0.0003119656,0.01000,0.336 23,mlp.experts.6.gate_proj,0.0051025631,0.01000,0.337 23,mlp.experts.7.gate_proj,0.0001383008,0.01000,0.336 23,mlp.experts.8.gate_proj,0.0001043042,0.01000,0.334 23,mlp.experts.9.gate_proj,0.0003181760,0.01000,0.336 23,mlp.experts.10.gate_proj,0.0011234193,0.01000,0.336 23,mlp.experts.11.gate_proj,0.0004781130,0.01000,0.337 23,mlp.experts.12.gate_proj,0.0004083309,0.01000,0.335 23,mlp.experts.13.gate_proj,0.0002512800,0.01000,0.334 23,mlp.experts.14.gate_proj,0.0001802935,0.01000,0.335 23,mlp.experts.15.gate_proj,0.0002442970,0.01000,0.337 23,mlp.experts.16.gate_proj,0.0005018141,0.01000,0.336 23,mlp.experts.17.gate_proj,0.0004299228,0.01000,0.334 23,mlp.experts.18.gate_proj,0.0002417097,0.01000,0.335 23,mlp.experts.19.gate_proj,0.0002604147,0.01000,0.338 23,mlp.experts.20.gate_proj,0.0009431104,0.01000,0.336 23,mlp.experts.21.gate_proj,0.0006228879,0.01000,0.334 23,mlp.experts.22.gate_proj,0.0001924120,0.01000,0.336 23,mlp.experts.23.gate_proj,0.0002536549,0.01000,0.336 23,mlp.experts.24.gate_proj,0.0187454361,0.01000,0.337 23,mlp.experts.25.gate_proj,0.0002052078,0.01000,0.336 23,mlp.experts.26.gate_proj,0.0037339931,0.01000,0.338 23,mlp.experts.27.gate_proj,0.0005331475,0.01000,0.336 23,mlp.experts.28.gate_proj,0.0001971778,0.01000,0.339 23,mlp.experts.29.gate_proj,0.0002036636,0.01000,0.335 23,mlp.experts.30.gate_proj,0.0001196277,0.01000,0.337 23,mlp.experts.31.gate_proj,0.0004971720,0.01000,0.349 23,mlp.experts.32.gate_proj,0.0001703321,0.01000,0.336 23,mlp.experts.33.gate_proj,0.0002384925,0.01000,0.335 23,mlp.experts.34.gate_proj,0.0012274648,0.01000,0.337 23,mlp.experts.35.gate_proj,0.0003024665,0.01000,0.337 23,mlp.experts.36.gate_proj,0.0002065561,0.01000,0.336 23,mlp.experts.37.gate_proj,0.0010775426,0.01000,0.336 23,mlp.experts.38.gate_proj,0.0002721696,0.01000,0.337 23,mlp.experts.39.gate_proj,0.0009371422,0.01000,0.337 23,mlp.experts.40.gate_proj,0.0002436298,0.01000,0.337 23,mlp.experts.41.gate_proj,0.0004728612,0.01000,0.337 23,mlp.experts.42.gate_proj,0.0004443144,0.01000,0.337 23,mlp.experts.43.gate_proj,0.0005823152,0.01000,0.337 23,mlp.experts.44.gate_proj,0.0002524613,0.01000,0.338 23,mlp.experts.45.gate_proj,0.0001676317,0.01000,0.338 23,mlp.experts.46.gate_proj,0.0010832949,0.01000,0.336 23,mlp.experts.47.gate_proj,0.0023440995,0.01000,0.339 23,mlp.experts.48.gate_proj,0.0006918773,0.01000,0.335 23,mlp.experts.49.gate_proj,0.0004173599,0.01000,0.336 23,mlp.experts.50.gate_proj,0.0003860662,0.01000,0.337 23,mlp.experts.51.gate_proj,0.0002524970,0.01000,0.336 23,mlp.experts.52.gate_proj,0.0008464437,0.01000,0.337 23,mlp.experts.53.gate_proj,0.0002080802,0.01000,0.336 23,mlp.experts.54.gate_proj,0.0007691859,0.01000,0.336 23,mlp.experts.55.gate_proj,0.0019891174,0.01000,0.339 23,mlp.experts.56.gate_proj,0.0003406622,0.01000,0.339 23,mlp.experts.57.gate_proj,0.0010292500,0.01000,0.338 23,mlp.experts.58.gate_proj,0.0001860731,0.01000,0.341 23,mlp.experts.59.gate_proj,0.0001166935,0.01000,0.338 23,mlp.experts.60.gate_proj,0.0360434749,0.01000,0.340 23,mlp.experts.61.gate_proj,0.0017129182,0.01000,0.337 23,mlp.experts.62.gate_proj,0.0002269491,0.01000,0.336 23,mlp.experts.63.gate_proj,0.0003146501,0.01000,0.336 23,mlp.experts.0.up_proj,0.0003626929,0.01000,0.357 23,mlp.experts.1.up_proj,0.0002278280,0.01000,0.343 23,mlp.experts.2.up_proj,0.0001498605,0.01000,0.345 23,mlp.experts.3.up_proj,0.0004592346,0.01000,0.345 23,mlp.experts.4.up_proj,0.0001516111,0.01000,0.346 23,mlp.experts.5.up_proj,0.0002990295,0.01000,0.346 23,mlp.experts.6.up_proj,0.0048670513,0.01000,0.343 23,mlp.experts.7.up_proj,0.0001346430,0.01000,0.344 23,mlp.experts.8.up_proj,0.0000966268,0.01000,0.347 23,mlp.experts.9.up_proj,0.0002731863,0.01000,0.345 23,mlp.experts.10.up_proj,0.0010420892,0.01000,0.345 23,mlp.experts.11.up_proj,0.0004626865,0.01000,0.346 23,mlp.experts.12.up_proj,0.0003866949,0.01000,0.346 23,mlp.experts.13.up_proj,0.0002360661,0.01000,0.348 23,mlp.experts.14.up_proj,0.0001710553,0.01000,0.346 23,mlp.experts.15.up_proj,0.0002326965,0.01000,0.348 23,mlp.experts.16.up_proj,0.0004879615,0.01000,0.347 23,mlp.experts.17.up_proj,0.0003829372,0.01000,0.337 23,mlp.experts.18.up_proj,0.0002367617,0.01000,0.338 23,mlp.experts.19.up_proj,0.0002282871,0.01000,0.338 23,mlp.experts.20.up_proj,0.0009073246,0.01000,0.340 23,mlp.experts.21.up_proj,0.0005963586,0.01000,0.338 23,mlp.experts.22.up_proj,0.0001727574,0.01000,0.338 23,mlp.experts.23.up_proj,0.0002599865,0.01000,0.340 23,mlp.experts.24.up_proj,0.0161238701,0.01000,0.338 23,mlp.experts.25.up_proj,0.0001978355,0.01000,0.340 23,mlp.experts.26.up_proj,0.0031812093,0.01000,0.343 23,mlp.experts.27.up_proj,0.0004896715,0.01000,0.336 23,mlp.experts.28.up_proj,0.0001897706,0.01000,0.339 23,mlp.experts.29.up_proj,0.0001938834,0.01000,0.337 23,mlp.experts.30.up_proj,0.0001147614,0.01000,0.334 23,mlp.experts.31.up_proj,0.0004676891,0.01000,0.336 23,mlp.experts.32.up_proj,0.0001466308,0.01000,0.335 23,mlp.experts.33.up_proj,0.0002306778,0.01000,0.335 23,mlp.experts.34.up_proj,0.0011826385,0.01000,0.336 23,mlp.experts.35.up_proj,0.0002959085,0.01000,0.335 23,mlp.experts.36.up_proj,0.0001786644,0.01000,0.336 23,mlp.experts.37.up_proj,0.0009788820,0.01000,0.336 23,mlp.experts.38.up_proj,0.0002701357,0.01000,0.335 23,mlp.experts.39.up_proj,0.0009194704,0.01000,0.337 23,mlp.experts.40.up_proj,0.0002295103,0.01000,0.337 23,mlp.experts.41.up_proj,0.0004537881,0.01000,0.337 23,mlp.experts.42.up_proj,0.0004303883,0.01000,0.336 23,mlp.experts.43.up_proj,0.0005419252,0.01000,0.335 23,mlp.experts.44.up_proj,0.0002353380,0.01000,0.336 23,mlp.experts.45.up_proj,0.0001508747,0.01000,0.335 23,mlp.experts.46.up_proj,0.0010745507,0.01000,0.336 23,mlp.experts.47.up_proj,0.0020067899,0.01000,0.335 23,mlp.experts.48.up_proj,0.0006340900,0.01000,0.336 23,mlp.experts.49.up_proj,0.0003953957,0.01000,0.334 23,mlp.experts.50.up_proj,0.0003474762,0.01000,0.335 23,mlp.experts.51.up_proj,0.0002323836,0.01000,0.334 23,mlp.experts.52.up_proj,0.0008113635,0.01000,0.335 23,mlp.experts.53.up_proj,0.0001951970,0.01000,0.336 23,mlp.experts.54.up_proj,0.0007738848,0.01000,0.335 23,mlp.experts.55.up_proj,0.0019896476,0.01000,0.335 23,mlp.experts.56.up_proj,0.0003070363,0.01000,0.336 23,mlp.experts.57.up_proj,0.0008755147,0.01000,0.336 23,mlp.experts.58.up_proj,0.0001788652,0.01000,0.335 23,mlp.experts.59.up_proj,0.0001073992,0.01000,0.336 23,mlp.experts.60.up_proj,0.0326032335,0.01000,0.335 23,mlp.experts.61.up_proj,0.0015476670,0.01000,0.333 23,mlp.experts.62.up_proj,0.0002180719,0.01000,0.334 23,mlp.experts.63.up_proj,0.0002830957,0.01000,0.335 23,mlp.experts.0.down_proj,0.0001801862,0.01000,0.242 23,mlp.experts.1.down_proj,0.0001043721,0.01000,0.230 23,mlp.experts.2.down_proj,0.0000684540,0.01000,0.230 23,mlp.experts.3.down_proj,0.0001794594,0.01000,0.236 23,mlp.experts.4.down_proj,0.0000673248,0.01000,0.235 23,mlp.experts.5.down_proj,0.0001597107,0.01000,0.229 23,mlp.experts.6.down_proj,0.0023416037,0.01000,0.230 23,mlp.experts.7.down_proj,0.0000724241,0.01000,0.230 23,mlp.experts.8.down_proj,0.0000537124,0.01000,0.230 23,mlp.experts.9.down_proj,0.0001103421,0.01000,0.230 23,mlp.experts.10.down_proj,0.0004688935,0.01000,0.230 23,mlp.experts.11.down_proj,0.0002245457,0.01000,0.230 23,mlp.experts.12.down_proj,0.0001600225,0.01000,0.230 23,mlp.experts.13.down_proj,0.0001241098,0.01000,0.231 23,mlp.experts.14.down_proj,0.0000900318,0.01000,0.231 23,mlp.experts.15.down_proj,0.0001250903,0.01000,0.231 23,mlp.experts.16.down_proj,0.0002518051,0.01000,0.232 23,mlp.experts.17.down_proj,0.0001690732,0.01000,0.231 23,mlp.experts.18.down_proj,0.0001062223,0.01000,0.233 23,mlp.experts.19.down_proj,0.0001258897,0.01000,0.232 23,mlp.experts.20.down_proj,0.0004050588,0.01000,0.230 23,mlp.experts.21.down_proj,0.0002682190,0.01000,0.231 23,mlp.experts.22.down_proj,0.0000825772,0.01000,0.232 23,mlp.experts.23.down_proj,0.0001097080,0.01000,0.231 23,mlp.experts.24.down_proj,0.0083566417,0.01000,0.235 23,mlp.experts.25.down_proj,0.0000811241,0.01000,0.234 23,mlp.experts.26.down_proj,0.0019373034,0.01000,0.232 23,mlp.experts.27.down_proj,0.0002422194,0.01000,0.230 23,mlp.experts.28.down_proj,0.0000877777,0.01000,0.230 23,mlp.experts.29.down_proj,0.0000829558,0.01000,0.234 23,mlp.experts.30.down_proj,0.0000575656,0.01000,0.232 23,mlp.experts.31.down_proj,0.0001950647,0.01000,0.232 23,mlp.experts.32.down_proj,0.0000754775,0.01000,0.233 23,mlp.experts.33.down_proj,0.0001122956,0.01000,0.231 23,mlp.experts.34.down_proj,0.0005360389,0.01000,0.233 23,mlp.experts.35.down_proj,0.0001445424,0.01000,0.230 23,mlp.experts.36.down_proj,0.0000803595,0.01000,0.230 23,mlp.experts.37.down_proj,0.0003940918,0.01000,0.230 23,mlp.experts.38.down_proj,0.0001446810,0.01000,0.229 23,mlp.experts.39.down_proj,0.0004844141,0.01000,0.230 23,mlp.experts.40.down_proj,0.0001151205,0.01000,0.229 23,mlp.experts.41.down_proj,0.0002181504,0.01000,0.230 23,mlp.experts.42.down_proj,0.0002142512,0.01000,0.231 23,mlp.experts.43.down_proj,0.0002459187,0.01000,0.233 23,mlp.experts.44.down_proj,0.0000919596,0.01000,0.231 23,mlp.experts.45.down_proj,0.0000798814,0.01000,0.231 23,mlp.experts.46.down_proj,0.0004867549,0.01000,0.230 23,mlp.experts.47.down_proj,0.0011863877,0.01000,0.231 23,mlp.experts.48.down_proj,0.0002629280,0.01000,0.229 23,mlp.experts.49.down_proj,0.0002224332,0.01000,0.229 23,mlp.experts.50.down_proj,0.0001937262,0.01000,0.230 23,mlp.experts.51.down_proj,0.0001418478,0.01000,0.230 23,mlp.experts.52.down_proj,0.0003775226,0.01000,0.231 23,mlp.experts.53.down_proj,0.0000878551,0.01000,0.230 23,mlp.experts.54.down_proj,0.0003395601,0.01000,0.230 23,mlp.experts.55.down_proj,0.0009333564,0.01000,0.230 23,mlp.experts.56.down_proj,0.0001503065,0.01000,0.230 23,mlp.experts.57.down_proj,0.0004348740,0.01000,0.230 23,mlp.experts.58.down_proj,0.0000923762,0.01000,0.230 23,mlp.experts.59.down_proj,0.0000584749,0.01000,0.230 23,mlp.experts.60.down_proj,0.0113301712,0.01000,0.231 23,mlp.experts.61.down_proj,0.0007183390,0.01000,0.231 23,mlp.experts.62.down_proj,0.0001263382,0.01000,0.230 23,mlp.experts.63.down_proj,0.0001621681,0.01000,0.230 23,mlp.shared_experts.gate_proj,0.0000369234,0.01000,0.345 23,mlp.shared_experts.up_proj,0.0000298005,0.01000,0.334 23,mlp.shared_experts.down_proj,0.0000034581,0.01000,0.475 24,self_attn.kv_a_proj_with_mqa,0.0000081309,0.01000,0.348 24,self_attn.kv_b_proj,0.0001295750,0.01000,0.097 24,self_attn.o_proj,0.0000047034,0.01000,0.345 24,mlp.experts.0.gate_proj,0.0009079751,0.01000,0.347 24,mlp.experts.1.gate_proj,0.0006120252,0.01000,0.338 24,mlp.experts.2.gate_proj,0.0030943439,0.01000,0.336 24,mlp.experts.3.gate_proj,0.0002888023,0.01000,0.335 24,mlp.experts.4.gate_proj,0.0001846122,0.01000,0.338 24,mlp.experts.5.gate_proj,0.0007070510,0.01000,0.338 24,mlp.experts.6.gate_proj,0.0001434450,0.01000,0.336 24,mlp.experts.7.gate_proj,0.0004027271,0.01000,0.338 24,mlp.experts.8.gate_proj,0.0009343131,0.01000,0.336 24,mlp.experts.9.gate_proj,0.0003528599,0.01000,0.337 24,mlp.experts.10.gate_proj,0.0008955385,0.01000,0.337 24,mlp.experts.11.gate_proj,0.0021898393,0.01000,0.336 24,mlp.experts.12.gate_proj,0.0004479406,0.01000,0.335 24,mlp.experts.13.gate_proj,0.0001079331,0.01000,0.336 24,mlp.experts.14.gate_proj,0.0002499359,0.01000,0.337 24,mlp.experts.15.gate_proj,0.0021021610,0.01000,0.339 24,mlp.experts.16.gate_proj,0.0183132960,0.01000,0.337 24,mlp.experts.17.gate_proj,0.0002586127,0.01000,0.337 24,mlp.experts.18.gate_proj,0.0001745979,0.01000,0.336 24,mlp.experts.19.gate_proj,0.0008513718,0.01000,0.337 24,mlp.experts.20.gate_proj,0.0001709178,0.01000,0.335 24,mlp.experts.21.gate_proj,0.0004111296,0.01000,0.335 24,mlp.experts.22.gate_proj,0.0007191733,0.01000,0.337 24,mlp.experts.23.gate_proj,0.0001823234,0.01000,0.336 24,mlp.experts.24.gate_proj,0.0001296288,0.01000,0.335 24,mlp.experts.25.gate_proj,0.0001998124,0.01000,0.336 24,mlp.experts.26.gate_proj,0.0003846341,0.01000,0.336 24,mlp.experts.27.gate_proj,0.0002795647,0.01000,0.337 24,mlp.experts.28.gate_proj,0.0022193878,0.01000,0.338 24,mlp.experts.29.gate_proj,0.0012556427,0.01000,0.335 24,mlp.experts.30.gate_proj,0.0003494921,0.01000,0.336 24,mlp.experts.31.gate_proj,0.0005146195,0.01000,0.336 24,mlp.experts.32.gate_proj,0.0020012019,0.01000,0.336 24,mlp.experts.33.gate_proj,0.0001865271,0.01000,0.336 24,mlp.experts.34.gate_proj,0.0006046310,0.01000,0.336 24,mlp.experts.35.gate_proj,0.0003037553,0.01000,0.336 24,mlp.experts.36.gate_proj,0.0002977102,0.01000,0.336 24,mlp.experts.37.gate_proj,0.0005278589,0.01000,0.337 24,mlp.experts.38.gate_proj,0.0002524166,0.01000,0.337 24,mlp.experts.39.gate_proj,0.0005142678,0.01000,0.337 24,mlp.experts.40.gate_proj,0.0010787161,0.01000,0.339 24,mlp.experts.41.gate_proj,0.0001645394,0.01000,0.339 24,mlp.experts.42.gate_proj,0.0005608645,0.01000,0.339 24,mlp.experts.43.gate_proj,0.0003979355,0.01000,0.337 24,mlp.experts.44.gate_proj,0.0015478844,0.01000,0.337 24,mlp.experts.45.gate_proj,0.0004407992,0.01000,0.337 24,mlp.experts.46.gate_proj,0.0006725039,0.01000,0.337 24,mlp.experts.47.gate_proj,0.0003634610,0.01000,0.343 24,mlp.experts.48.gate_proj,0.0002267079,0.01000,0.336 24,mlp.experts.49.gate_proj,0.0005110195,0.01000,0.337 24,mlp.experts.50.gate_proj,0.0004538862,0.01000,0.345 24,mlp.experts.51.gate_proj,0.0007305612,0.01000,0.337 24,mlp.experts.52.gate_proj,0.0060250568,0.01000,0.337 24,mlp.experts.53.gate_proj,0.0002009787,0.01000,0.339 24,mlp.experts.54.gate_proj,0.0005451281,0.01000,0.338 24,mlp.experts.55.gate_proj,0.0009714771,0.01000,0.337 24,mlp.experts.56.gate_proj,0.0002918436,0.01000,0.338 24,mlp.experts.57.gate_proj,0.0001872035,0.01000,0.335 24,mlp.experts.58.gate_proj,0.0012879521,0.01000,0.335 24,mlp.experts.59.gate_proj,0.0009563731,0.01000,0.335 24,mlp.experts.60.gate_proj,0.0002628959,0.01000,0.335 24,mlp.experts.61.gate_proj,0.0003553949,0.01000,0.339 24,mlp.experts.62.gate_proj,0.0003169424,0.01000,0.339 24,mlp.experts.63.gate_proj,0.0002857538,0.01000,0.337 24,mlp.experts.0.up_proj,0.0008796504,0.01000,0.346 24,mlp.experts.1.up_proj,0.0005851761,0.01000,0.337 24,mlp.experts.2.up_proj,0.0029750849,0.01000,0.337 24,mlp.experts.3.up_proj,0.0002785372,0.01000,0.337 24,mlp.experts.4.up_proj,0.0001806495,0.01000,0.338 24,mlp.experts.5.up_proj,0.0007048210,0.01000,0.337 24,mlp.experts.6.up_proj,0.0001313795,0.01000,0.335 24,mlp.experts.7.up_proj,0.0004058888,0.01000,0.336 24,mlp.experts.8.up_proj,0.0009335167,0.01000,0.335 24,mlp.experts.9.up_proj,0.0003418075,0.01000,0.335 24,mlp.experts.10.up_proj,0.0008958480,0.01000,0.335 24,mlp.experts.11.up_proj,0.0021575699,0.01000,0.336 24,mlp.experts.12.up_proj,0.0004257982,0.01000,0.336 24,mlp.experts.13.up_proj,0.0001026139,0.01000,0.336 24,mlp.experts.14.up_proj,0.0002422184,0.01000,0.336 24,mlp.experts.15.up_proj,0.0019047473,0.01000,0.337 24,mlp.experts.16.up_proj,0.0158640436,0.01000,0.338 24,mlp.experts.17.up_proj,0.0002513973,0.01000,0.337 24,mlp.experts.18.up_proj,0.0001589267,0.01000,0.336 24,mlp.experts.19.up_proj,0.0007839552,0.01000,0.337 24,mlp.experts.20.up_proj,0.0001623533,0.01000,0.337 24,mlp.experts.21.up_proj,0.0003888699,0.01000,0.336 24,mlp.experts.22.up_proj,0.0006925589,0.01000,0.338 24,mlp.experts.23.up_proj,0.0001766275,0.01000,0.337 24,mlp.experts.24.up_proj,0.0001225178,0.01000,0.336 24,mlp.experts.25.up_proj,0.0001901979,0.01000,0.335 24,mlp.experts.26.up_proj,0.0003686475,0.01000,0.335 24,mlp.experts.27.up_proj,0.0002763350,0.01000,0.337 24,mlp.experts.28.up_proj,0.0021780685,0.01000,0.337 24,mlp.experts.29.up_proj,0.0012578669,0.01000,0.336 24,mlp.experts.30.up_proj,0.0003455257,0.01000,0.337 24,mlp.experts.31.up_proj,0.0004891961,0.01000,0.337 24,mlp.experts.32.up_proj,0.0020146660,0.01000,0.336 24,mlp.experts.33.up_proj,0.0001774731,0.01000,0.338 24,mlp.experts.34.up_proj,0.0005834633,0.01000,0.339 24,mlp.experts.35.up_proj,0.0002777877,0.01000,0.337 24,mlp.experts.36.up_proj,0.0002873248,0.01000,0.336 24,mlp.experts.37.up_proj,0.0005156380,0.01000,0.336 24,mlp.experts.38.up_proj,0.0002384852,0.01000,0.336 24,mlp.experts.39.up_proj,0.0004402057,0.01000,0.358 24,mlp.experts.40.up_proj,0.0010391255,0.01000,0.339 24,mlp.experts.41.up_proj,0.0001510357,0.01000,0.339 24,mlp.experts.42.up_proj,0.0005327693,0.01000,0.339 24,mlp.experts.43.up_proj,0.0003716136,0.01000,0.338 24,mlp.experts.44.up_proj,0.0015117105,0.01000,0.339 24,mlp.experts.45.up_proj,0.0004338939,0.01000,0.337 24,mlp.experts.46.up_proj,0.0006790996,0.01000,0.339 24,mlp.experts.47.up_proj,0.0003043397,0.01000,0.336 24,mlp.experts.48.up_proj,0.0002177303,0.01000,0.337 24,mlp.experts.49.up_proj,0.0004895527,0.01000,0.337 24,mlp.experts.50.up_proj,0.0004328193,0.01000,0.335 24,mlp.experts.51.up_proj,0.0007095083,0.01000,0.334 24,mlp.experts.52.up_proj,0.0055848351,0.01000,0.335 24,mlp.experts.53.up_proj,0.0001955796,0.01000,0.337 24,mlp.experts.54.up_proj,0.0004997202,0.01000,0.334 24,mlp.experts.55.up_proj,0.0009390191,0.01000,0.336 24,mlp.experts.56.up_proj,0.0002867269,0.01000,0.336 24,mlp.experts.57.up_proj,0.0001763609,0.01000,0.335 24,mlp.experts.58.up_proj,0.0011157679,0.01000,0.334 24,mlp.experts.59.up_proj,0.0008973828,0.01000,0.334 24,mlp.experts.60.up_proj,0.0002417572,0.01000,0.336 24,mlp.experts.61.up_proj,0.0003570266,0.01000,0.335 24,mlp.experts.62.up_proj,0.0003057007,0.01000,0.335 24,mlp.experts.63.up_proj,0.0002671540,0.01000,0.334 24,mlp.experts.0.down_proj,0.0004337720,0.01000,0.241 24,mlp.experts.1.down_proj,0.0003881737,0.01000,0.232 24,mlp.experts.2.down_proj,0.0018080914,0.01000,0.233 24,mlp.experts.3.down_proj,0.0001631622,0.01000,0.232 24,mlp.experts.4.down_proj,0.0001274510,0.01000,0.235 24,mlp.experts.5.down_proj,0.0004433532,0.01000,0.237 24,mlp.experts.6.down_proj,0.0000919379,0.01000,0.230 24,mlp.experts.7.down_proj,0.0002477798,0.01000,0.233 24,mlp.experts.8.down_proj,0.0005632844,0.01000,0.233 24,mlp.experts.9.down_proj,0.0001894870,0.01000,0.232 24,mlp.experts.10.down_proj,0.0005245564,0.01000,0.230 24,mlp.experts.11.down_proj,0.0012706982,0.01000,0.230 24,mlp.experts.12.down_proj,0.0002719429,0.01000,0.232 24,mlp.experts.13.down_proj,0.0000488318,0.01000,0.231 24,mlp.experts.14.down_proj,0.0001297635,0.01000,0.231 24,mlp.experts.15.down_proj,0.0011514087,0.01000,0.232 24,mlp.experts.16.down_proj,0.0116709822,0.01000,0.230 24,mlp.experts.17.down_proj,0.0001482989,0.01000,0.231 24,mlp.experts.18.down_proj,0.0000934537,0.01000,0.233 24,mlp.experts.19.down_proj,0.0005300723,0.01000,0.231 24,mlp.experts.20.down_proj,0.0000924589,0.01000,0.233 24,mlp.experts.21.down_proj,0.0002419811,0.01000,0.230 24,mlp.experts.22.down_proj,0.0004625744,0.01000,0.232 24,mlp.experts.23.down_proj,0.0000971661,0.01000,0.231 24,mlp.experts.24.down_proj,0.0000667771,0.01000,0.232 24,mlp.experts.25.down_proj,0.0001063431,0.01000,0.232 24,mlp.experts.26.down_proj,0.0002335299,0.01000,0.232 24,mlp.experts.27.down_proj,0.0001536393,0.01000,0.231 24,mlp.experts.28.down_proj,0.0013997287,0.01000,0.235 24,mlp.experts.29.down_proj,0.0007750974,0.01000,0.232 24,mlp.experts.30.down_proj,0.0002105541,0.01000,0.232 24,mlp.experts.31.down_proj,0.0002526464,0.01000,0.230 24,mlp.experts.32.down_proj,0.0011257808,0.01000,0.229 24,mlp.experts.33.down_proj,0.0001038936,0.01000,0.229 24,mlp.experts.34.down_proj,0.0004149754,0.01000,0.229 24,mlp.experts.35.down_proj,0.0001411487,0.01000,0.230 24,mlp.experts.36.down_proj,0.0001771020,0.01000,0.230 24,mlp.experts.37.down_proj,0.0003176121,0.01000,0.230 24,mlp.experts.38.down_proj,0.0001493335,0.01000,0.230 24,mlp.experts.39.down_proj,0.0002694209,0.01000,0.228 24,mlp.experts.40.down_proj,0.0005520590,0.01000,0.229 24,mlp.experts.41.down_proj,0.0000889120,0.01000,0.230 24,mlp.experts.42.down_proj,0.0002950562,0.01000,0.230 24,mlp.experts.43.down_proj,0.0002547921,0.01000,0.231 24,mlp.experts.44.down_proj,0.0009937106,0.01000,0.231 24,mlp.experts.45.down_proj,0.0002377401,0.01000,0.229 24,mlp.experts.46.down_proj,0.0003989287,0.01000,0.230 24,mlp.experts.47.down_proj,0.0002287596,0.01000,0.229 24,mlp.experts.48.down_proj,0.0001196159,0.01000,0.229 24,mlp.experts.49.down_proj,0.0002635142,0.01000,0.229 24,mlp.experts.50.down_proj,0.0002625486,0.01000,0.229 24,mlp.experts.51.down_proj,0.0004038889,0.01000,0.230 24,mlp.experts.52.down_proj,0.0030145806,0.01000,0.230 24,mlp.experts.53.down_proj,0.0001348099,0.01000,0.229 24,mlp.experts.54.down_proj,0.0003589999,0.01000,0.229 24,mlp.experts.55.down_proj,0.0004660629,0.01000,0.230 24,mlp.experts.56.down_proj,0.0001555809,0.01000,0.229 24,mlp.experts.57.down_proj,0.0001226018,0.01000,0.229 24,mlp.experts.58.down_proj,0.0005721171,0.01000,0.229 24,mlp.experts.59.down_proj,0.0005707091,0.01000,0.229 24,mlp.experts.60.down_proj,0.0001813058,0.01000,0.229 24,mlp.experts.61.down_proj,0.0002022831,0.01000,0.229 24,mlp.experts.62.down_proj,0.0001417740,0.01000,0.241 24,mlp.experts.63.down_proj,0.0001420817,0.01000,0.230 24,mlp.shared_experts.gate_proj,0.0000514205,0.01000,0.345 24,mlp.shared_experts.up_proj,0.0000363148,0.01000,0.335 24,mlp.shared_experts.down_proj,0.0000046571,0.01000,0.470 25,self_attn.kv_a_proj_with_mqa,0.0000078264,0.01000,0.348 25,self_attn.kv_b_proj,0.0001276474,0.01000,0.097 25,self_attn.o_proj,0.0000063819,0.01000,0.343 25,mlp.experts.0.gate_proj,0.0014160929,0.01000,0.345 25,mlp.experts.1.gate_proj,0.0002619773,0.01000,0.338 25,mlp.experts.2.gate_proj,0.0002225956,0.01000,0.338 25,mlp.experts.3.gate_proj,0.0001094307,0.01000,0.341 25,mlp.experts.4.gate_proj,0.0008274527,0.01000,0.342 25,mlp.experts.5.gate_proj,0.0014291878,0.01000,0.339 25,mlp.experts.6.gate_proj,0.0004483601,0.01000,0.338 25,mlp.experts.7.gate_proj,0.0001860456,0.01000,0.339 25,mlp.experts.8.gate_proj,0.0002555963,0.01000,0.349 25,mlp.experts.9.gate_proj,0.0008227348,0.01000,0.338 25,mlp.experts.10.gate_proj,0.0002780168,0.01000,0.336 25,mlp.experts.11.gate_proj,0.0010200134,0.01000,0.338 25,mlp.experts.12.gate_proj,0.0002728019,0.01000,0.337 25,mlp.experts.13.gate_proj,0.0008961502,0.01000,0.338 25,mlp.experts.14.gate_proj,0.0002530653,0.01000,0.338 25,mlp.experts.15.gate_proj,0.0003981738,0.01000,0.340 25,mlp.experts.16.gate_proj,0.0017716978,0.01000,0.339 25,mlp.experts.17.gate_proj,0.0012767806,0.01000,0.340 25,mlp.experts.18.gate_proj,0.0001217836,0.01000,0.344 25,mlp.experts.19.gate_proj,0.0021247198,0.01000,0.336 25,mlp.experts.20.gate_proj,0.0008533958,0.01000,0.340 25,mlp.experts.21.gate_proj,0.0009905944,0.01000,0.338 25,mlp.experts.22.gate_proj,0.0033494178,0.01000,0.336 25,mlp.experts.23.gate_proj,0.0008614676,0.01000,0.334 25,mlp.experts.24.gate_proj,0.0013761328,0.01000,0.334 25,mlp.experts.25.gate_proj,0.0002501389,0.01000,0.335 25,mlp.experts.26.gate_proj,0.0006290382,0.01000,0.336 25,mlp.experts.27.gate_proj,0.0002148004,0.01000,0.334 25,mlp.experts.28.gate_proj,0.0015999201,0.01000,0.338 25,mlp.experts.29.gate_proj,0.0017371742,0.01000,0.334 25,mlp.experts.30.gate_proj,0.0004498645,0.01000,0.335 25,mlp.experts.31.gate_proj,0.0040567445,0.01000,0.336 25,mlp.experts.32.gate_proj,0.0013867344,0.01000,0.338 25,mlp.experts.33.gate_proj,0.0007250458,0.01000,0.336 25,mlp.experts.34.gate_proj,0.0005378423,0.01000,0.335 25,mlp.experts.35.gate_proj,0.0007401646,0.01000,0.339 25,mlp.experts.36.gate_proj,0.0008616054,0.01000,0.336 25,mlp.experts.37.gate_proj,0.0001916793,0.01000,0.335 25,mlp.experts.38.gate_proj,0.0003026717,0.01000,0.336 25,mlp.experts.39.gate_proj,0.0020242522,0.01000,0.338 25,mlp.experts.40.gate_proj,0.0005491048,0.01000,0.336 25,mlp.experts.41.gate_proj,0.0004507046,0.01000,0.335 25,mlp.experts.42.gate_proj,0.0005118224,0.01000,0.335 25,mlp.experts.43.gate_proj,0.0012676814,0.01000,0.336 25,mlp.experts.44.gate_proj,0.0002426296,0.01000,0.336 25,mlp.experts.45.gate_proj,0.0002681382,0.01000,0.336 25,mlp.experts.46.gate_proj,0.0004904258,0.01000,0.333 25,mlp.experts.47.gate_proj,0.0004599555,0.01000,0.349 25,mlp.experts.48.gate_proj,0.0003811106,0.01000,0.336 25,mlp.experts.49.gate_proj,0.0001919394,0.01000,0.336 25,mlp.experts.50.gate_proj,0.0004002982,0.01000,0.336 25,mlp.experts.51.gate_proj,0.0003050327,0.01000,0.334 25,mlp.experts.52.gate_proj,0.0003561939,0.01000,0.333 25,mlp.experts.53.gate_proj,0.0006122548,0.01000,0.334 25,mlp.experts.54.gate_proj,0.0005375274,0.01000,0.335 25,mlp.experts.55.gate_proj,0.0004130212,0.01000,0.335 25,mlp.experts.56.gate_proj,0.0023138324,0.01000,0.335 25,mlp.experts.57.gate_proj,0.0006404100,0.01000,0.334 25,mlp.experts.58.gate_proj,0.0002088401,0.01000,0.334 25,mlp.experts.59.gate_proj,0.0003617585,0.01000,0.336 25,mlp.experts.60.gate_proj,0.0010892976,0.01000,0.334 25,mlp.experts.61.gate_proj,0.0008972068,0.01000,0.334 25,mlp.experts.62.gate_proj,0.0002202946,0.01000,0.335 25,mlp.experts.63.gate_proj,0.0005478218,0.01000,0.334 25,mlp.experts.0.up_proj,0.0013370664,0.01000,0.351 25,mlp.experts.1.up_proj,0.0002379365,0.01000,0.337 25,mlp.experts.2.up_proj,0.0002072177,0.01000,0.336 25,mlp.experts.3.up_proj,0.0001042134,0.01000,0.336 25,mlp.experts.4.up_proj,0.0007803054,0.01000,0.337 25,mlp.experts.5.up_proj,0.0014290320,0.01000,0.337 25,mlp.experts.6.up_proj,0.0004337568,0.01000,0.337 25,mlp.experts.7.up_proj,0.0001774441,0.01000,0.334 25,mlp.experts.8.up_proj,0.0002439926,0.01000,0.336 25,mlp.experts.9.up_proj,0.0007971061,0.01000,0.335 25,mlp.experts.10.up_proj,0.0002625255,0.01000,0.335 25,mlp.experts.11.up_proj,0.0009592134,0.01000,0.335 25,mlp.experts.12.up_proj,0.0002627245,0.01000,0.337 25,mlp.experts.13.up_proj,0.0007276856,0.01000,0.337 25,mlp.experts.14.up_proj,0.0002384958,0.01000,0.337 25,mlp.experts.15.up_proj,0.0003961537,0.01000,0.337 25,mlp.experts.16.up_proj,0.0016065406,0.01000,0.337 25,mlp.experts.17.up_proj,0.0012860742,0.01000,0.336 25,mlp.experts.18.up_proj,0.0001125083,0.01000,0.336 25,mlp.experts.19.up_proj,0.0019961785,0.01000,0.336 25,mlp.experts.20.up_proj,0.0007942769,0.01000,0.337 25,mlp.experts.21.up_proj,0.0009522927,0.01000,0.336 25,mlp.experts.22.up_proj,0.0031982031,0.01000,0.335 25,mlp.experts.23.up_proj,0.0007351224,0.01000,0.335 25,mlp.experts.24.up_proj,0.0012465112,0.01000,0.337 25,mlp.experts.25.up_proj,0.0002240191,0.01000,0.337 25,mlp.experts.26.up_proj,0.0005245421,0.01000,0.338 25,mlp.experts.27.up_proj,0.0002124198,0.01000,0.336 25,mlp.experts.28.up_proj,0.0015399657,0.01000,0.337 25,mlp.experts.29.up_proj,0.0016568030,0.01000,0.335 25,mlp.experts.30.up_proj,0.0004183405,0.01000,0.335 25,mlp.experts.31.up_proj,0.0040043421,0.01000,0.338 25,mlp.experts.32.up_proj,0.0011937205,0.01000,0.335 25,mlp.experts.33.up_proj,0.0007147261,0.01000,0.336 25,mlp.experts.34.up_proj,0.0005176664,0.01000,0.337 25,mlp.experts.35.up_proj,0.0007020514,0.01000,0.337 25,mlp.experts.36.up_proj,0.0008259066,0.01000,0.336 25,mlp.experts.37.up_proj,0.0001688669,0.01000,0.338 25,mlp.experts.38.up_proj,0.0002895648,0.01000,0.337 25,mlp.experts.39.up_proj,0.0019011721,0.01000,0.336 25,mlp.experts.40.up_proj,0.0005302237,0.01000,0.337 25,mlp.experts.41.up_proj,0.0004316466,0.01000,0.336 25,mlp.experts.42.up_proj,0.0005034769,0.01000,0.336 25,mlp.experts.43.up_proj,0.0012135010,0.01000,0.337 25,mlp.experts.44.up_proj,0.0002224357,0.01000,0.336 25,mlp.experts.45.up_proj,0.0002603735,0.01000,0.336 25,mlp.experts.46.up_proj,0.0004335494,0.01000,0.335 25,mlp.experts.47.up_proj,0.0004226047,0.01000,0.336 25,mlp.experts.48.up_proj,0.0003779718,0.01000,0.336 25,mlp.experts.49.up_proj,0.0001817145,0.01000,0.338 25,mlp.experts.50.up_proj,0.0003679797,0.01000,0.337 25,mlp.experts.51.up_proj,0.0002919943,0.01000,0.337 25,mlp.experts.52.up_proj,0.0003568193,0.01000,0.335 25,mlp.experts.53.up_proj,0.0005908021,0.01000,0.335 25,mlp.experts.54.up_proj,0.0005273948,0.01000,0.340 25,mlp.experts.55.up_proj,0.0003989157,0.01000,0.339 25,mlp.experts.56.up_proj,0.0021786237,0.01000,0.337 25,mlp.experts.57.up_proj,0.0006190013,0.01000,0.336 25,mlp.experts.58.up_proj,0.0001883921,0.01000,0.340 25,mlp.experts.59.up_proj,0.0003570154,0.01000,0.339 25,mlp.experts.60.up_proj,0.0010354972,0.01000,0.340 25,mlp.experts.61.up_proj,0.0008278637,0.01000,0.337 25,mlp.experts.62.up_proj,0.0002066029,0.01000,0.340 25,mlp.experts.63.up_proj,0.0005260759,0.01000,0.338 25,mlp.experts.0.down_proj,0.0007364198,0.01000,0.242 25,mlp.experts.1.down_proj,0.0001829474,0.01000,0.230 25,mlp.experts.2.down_proj,0.0001279747,0.01000,0.230 25,mlp.experts.3.down_proj,0.0000798082,0.01000,0.230 25,mlp.experts.4.down_proj,0.0005371410,0.01000,0.231 25,mlp.experts.5.down_proj,0.0011035149,0.01000,0.231 25,mlp.experts.6.down_proj,0.0003036148,0.01000,0.230 25,mlp.experts.7.down_proj,0.0001469900,0.01000,0.232 25,mlp.experts.8.down_proj,0.0001639255,0.01000,0.232 25,mlp.experts.9.down_proj,0.0006805495,0.01000,0.231 25,mlp.experts.10.down_proj,0.0002100123,0.01000,0.232 25,mlp.experts.11.down_proj,0.0005239043,0.01000,0.233 25,mlp.experts.12.down_proj,0.0002078887,0.01000,0.232 25,mlp.experts.13.down_proj,0.0003644437,0.01000,0.234 25,mlp.experts.14.down_proj,0.0001582177,0.01000,0.231 25,mlp.experts.15.down_proj,0.0002761533,0.01000,0.234 25,mlp.experts.16.down_proj,0.0011003247,0.01000,0.231 25,mlp.experts.17.down_proj,0.0009247432,0.01000,0.231 25,mlp.experts.18.down_proj,0.0000754995,0.01000,0.231 25,mlp.experts.19.down_proj,0.0014897232,0.01000,0.231 25,mlp.experts.20.down_proj,0.0004923830,0.01000,0.231 25,mlp.experts.21.down_proj,0.0006286652,0.01000,0.232 25,mlp.experts.22.down_proj,0.0020082605,0.01000,0.234 25,mlp.experts.23.down_proj,0.0003806140,0.01000,0.233 25,mlp.experts.24.down_proj,0.0010539685,0.01000,0.240 25,mlp.experts.25.down_proj,0.0001362463,0.01000,0.232 25,mlp.experts.26.down_proj,0.0003740425,0.01000,0.232 25,mlp.experts.27.down_proj,0.0001222711,0.01000,0.232 25,mlp.experts.28.down_proj,0.0010633821,0.01000,0.236 25,mlp.experts.29.down_proj,0.0012241738,0.01000,0.230 25,mlp.experts.30.down_proj,0.0003100110,0.01000,0.231 25,mlp.experts.31.down_proj,0.0025359974,0.01000,0.231 25,mlp.experts.32.down_proj,0.0009531356,0.01000,0.230 25,mlp.experts.33.down_proj,0.0006225031,0.01000,0.230 25,mlp.experts.34.down_proj,0.0003647835,0.01000,0.229 25,mlp.experts.35.down_proj,0.0005504380,0.01000,0.229 25,mlp.experts.36.down_proj,0.0005640474,0.01000,0.229 25,mlp.experts.37.down_proj,0.0001370656,0.01000,0.228 25,mlp.experts.38.down_proj,0.0001690419,0.01000,0.229 25,mlp.experts.39.down_proj,0.0011282676,0.01000,0.232 25,mlp.experts.40.down_proj,0.0004272529,0.01000,0.241 25,mlp.experts.41.down_proj,0.0003080516,0.01000,0.229 25,mlp.experts.42.down_proj,0.0003352802,0.01000,0.230 25,mlp.experts.43.down_proj,0.0007452670,0.01000,0.231 25,mlp.experts.44.down_proj,0.0001541529,0.01000,0.231 25,mlp.experts.45.down_proj,0.0001816860,0.01000,0.231 25,mlp.experts.46.down_proj,0.0002529235,0.01000,0.231 25,mlp.experts.47.down_proj,0.0003505036,0.01000,0.230 25,mlp.experts.48.down_proj,0.0002862798,0.01000,0.232 25,mlp.experts.49.down_proj,0.0001333790,0.01000,0.231 25,mlp.experts.50.down_proj,0.0002918944,0.01000,0.230 25,mlp.experts.51.down_proj,0.0002035922,0.01000,0.230 25,mlp.experts.52.down_proj,0.0002333950,0.01000,0.230 25,mlp.experts.53.down_proj,0.0003595783,0.01000,0.229 25,mlp.experts.54.down_proj,0.0003713203,0.01000,0.229 25,mlp.experts.55.down_proj,0.0002580202,0.01000,0.229 25,mlp.experts.56.down_proj,0.0015528088,0.01000,0.229 25,mlp.experts.57.down_proj,0.0004100564,0.01000,0.229 25,mlp.experts.58.down_proj,0.0001504487,0.01000,0.230 25,mlp.experts.59.down_proj,0.0002586719,0.01000,0.230 25,mlp.experts.60.down_proj,0.0007797372,0.01000,0.230 25,mlp.experts.61.down_proj,0.0006632639,0.01000,0.229 25,mlp.experts.62.down_proj,0.0001882494,0.01000,0.230 25,mlp.experts.63.down_proj,0.0003685519,0.01000,0.230 25,mlp.shared_experts.gate_proj,0.0000737318,0.01000,0.347 25,mlp.shared_experts.up_proj,0.0000443153,0.01000,0.335 25,mlp.shared_experts.down_proj,0.0000059133,0.01000,0.473 26,self_attn.kv_a_proj_with_mqa,0.0000092990,0.01000,0.351 26,self_attn.kv_b_proj,0.0001043958,0.01000,0.100 26,self_attn.o_proj,0.0000078456,0.01000,0.348 26,mlp.experts.0.gate_proj,0.0001294137,0.01000,0.350 26,mlp.experts.1.gate_proj,0.0039056070,0.01000,0.337 26,mlp.experts.2.gate_proj,0.0007582005,0.01000,0.334 26,mlp.experts.3.gate_proj,0.0004490806,0.01000,0.336 26,mlp.experts.4.gate_proj,0.0002756955,0.01000,0.338 26,mlp.experts.5.gate_proj,0.0010641906,0.01000,0.336 26,mlp.experts.6.gate_proj,0.0004474290,0.01000,0.338 26,mlp.experts.7.gate_proj,0.0001863038,0.01000,0.339 26,mlp.experts.8.gate_proj,0.0002730934,0.01000,0.335 26,mlp.experts.9.gate_proj,0.0004732398,0.01000,0.337 26,mlp.experts.10.gate_proj,0.0012211578,0.01000,0.337 26,mlp.experts.11.gate_proj,0.0037696768,0.01000,0.335 26,mlp.experts.12.gate_proj,0.0002811692,0.01000,0.335 26,mlp.experts.13.gate_proj,0.0005439541,0.01000,0.336 26,mlp.experts.14.gate_proj,0.0002594174,0.01000,0.338 26,mlp.experts.15.gate_proj,0.0001156127,0.01000,0.335 26,mlp.experts.16.gate_proj,0.0017222333,0.01000,0.339 26,mlp.experts.17.gate_proj,0.0001606315,0.01000,0.338 26,mlp.experts.18.gate_proj,0.0005968673,0.01000,0.338 26,mlp.experts.19.gate_proj,0.0026713017,0.01000,0.338 26,mlp.experts.20.gate_proj,0.0018634310,0.01000,0.335 26,mlp.experts.21.gate_proj,0.0003375021,0.01000,0.341 26,mlp.experts.22.gate_proj,0.0006963426,0.01000,0.343 26,mlp.experts.23.gate_proj,0.0005423965,0.01000,0.338 26,mlp.experts.24.gate_proj,0.0010584356,0.01000,0.337 26,mlp.experts.25.gate_proj,0.0002800827,0.01000,0.337 26,mlp.experts.26.gate_proj,0.0006694017,0.01000,0.336 26,mlp.experts.27.gate_proj,0.0005455712,0.01000,0.339 26,mlp.experts.28.gate_proj,0.0008736787,0.01000,0.339 26,mlp.experts.29.gate_proj,0.0001496910,0.01000,0.336 26,mlp.experts.30.gate_proj,0.0039614447,0.01000,0.341 26,mlp.experts.31.gate_proj,0.0009838937,0.01000,0.341 26,mlp.experts.32.gate_proj,0.0006971416,0.01000,0.339 26,mlp.experts.33.gate_proj,0.0004670250,0.01000,0.338 26,mlp.experts.34.gate_proj,0.0001101791,0.01000,0.337 26,mlp.experts.35.gate_proj,0.0005975909,0.01000,0.334 26,mlp.experts.36.gate_proj,0.0000820577,0.01000,0.334 26,mlp.experts.37.gate_proj,0.0005018580,0.01000,0.335 26,mlp.experts.38.gate_proj,0.0019686680,0.01000,0.335 26,mlp.experts.39.gate_proj,0.0039433110,0.01000,0.337 26,mlp.experts.40.gate_proj,0.0004869035,0.01000,0.336 26,mlp.experts.41.gate_proj,0.0019423383,0.01000,0.336 26,mlp.experts.42.gate_proj,0.0040692572,0.01000,0.336 26,mlp.experts.43.gate_proj,0.0002075818,0.01000,0.336 26,mlp.experts.44.gate_proj,0.0004121341,0.01000,0.335 26,mlp.experts.45.gate_proj,0.0004059050,0.01000,0.335 26,mlp.experts.46.gate_proj,0.0003688652,0.01000,0.336 26,mlp.experts.47.gate_proj,0.0004887132,0.01000,0.335 26,mlp.experts.48.gate_proj,0.0002364023,0.01000,0.336 26,mlp.experts.49.gate_proj,0.0003895019,0.01000,0.336 26,mlp.experts.50.gate_proj,0.0008735755,0.01000,0.337 26,mlp.experts.51.gate_proj,0.0016665823,0.01000,0.336 26,mlp.experts.52.gate_proj,0.0000882799,0.01000,0.334 26,mlp.experts.53.gate_proj,0.0013633907,0.01000,0.334 26,mlp.experts.54.gate_proj,0.0003110695,0.01000,0.335 26,mlp.experts.55.gate_proj,0.0006625974,0.01000,0.335 26,mlp.experts.56.gate_proj,0.0024558343,0.01000,0.334 26,mlp.experts.57.gate_proj,0.0006339180,0.01000,0.335 26,mlp.experts.58.gate_proj,0.0005393560,0.01000,0.334 26,mlp.experts.59.gate_proj,0.0005114519,0.01000,0.335 26,mlp.experts.60.gate_proj,0.0001484010,0.01000,0.335 26,mlp.experts.61.gate_proj,0.0006917297,0.01000,0.334 26,mlp.experts.62.gate_proj,0.0013981060,0.01000,0.334 26,mlp.experts.63.gate_proj,0.0001058228,0.01000,0.335 26,mlp.experts.0.up_proj,0.0001149640,0.01000,0.350 26,mlp.experts.1.up_proj,0.0036469326,0.01000,0.338 26,mlp.experts.2.up_proj,0.0006815404,0.01000,0.337 26,mlp.experts.3.up_proj,0.0003965749,0.01000,0.335 26,mlp.experts.4.up_proj,0.0002600194,0.01000,0.335 26,mlp.experts.5.up_proj,0.0008725833,0.01000,0.336 26,mlp.experts.6.up_proj,0.0004038085,0.01000,0.337 26,mlp.experts.7.up_proj,0.0001671640,0.01000,0.335 26,mlp.experts.8.up_proj,0.0002529907,0.01000,0.335 26,mlp.experts.9.up_proj,0.0004227976,0.01000,0.335 26,mlp.experts.10.up_proj,0.0011027403,0.01000,0.336 26,mlp.experts.11.up_proj,0.0034319320,0.01000,0.337 26,mlp.experts.12.up_proj,0.0002623196,0.01000,0.337 26,mlp.experts.13.up_proj,0.0004797984,0.01000,0.334 26,mlp.experts.14.up_proj,0.0002010739,0.01000,0.335 26,mlp.experts.15.up_proj,0.0001004639,0.01000,0.336 26,mlp.experts.16.up_proj,0.0015579725,0.01000,0.338 26,mlp.experts.17.up_proj,0.0001411843,0.01000,0.338 26,mlp.experts.18.up_proj,0.0005643909,0.01000,0.336 26,mlp.experts.19.up_proj,0.0024853921,0.01000,0.335 26,mlp.experts.20.up_proj,0.0017205786,0.01000,0.336 26,mlp.experts.21.up_proj,0.0002945671,0.01000,0.335 26,mlp.experts.22.up_proj,0.0006272026,0.01000,0.337 26,mlp.experts.23.up_proj,0.0005400519,0.01000,0.336 26,mlp.experts.24.up_proj,0.0009256526,0.01000,0.336 26,mlp.experts.25.up_proj,0.0002543758,0.01000,0.336 26,mlp.experts.26.up_proj,0.0005760064,0.01000,0.338 26,mlp.experts.27.up_proj,0.0004551814,0.01000,0.335 26,mlp.experts.28.up_proj,0.0007460905,0.01000,0.335 26,mlp.experts.29.up_proj,0.0001307957,0.01000,0.337 26,mlp.experts.30.up_proj,0.0035443318,0.01000,0.337 26,mlp.experts.31.up_proj,0.0008758027,0.01000,0.338 26,mlp.experts.32.up_proj,0.0006305459,0.01000,0.335 26,mlp.experts.33.up_proj,0.0004029547,0.01000,0.337 26,mlp.experts.34.up_proj,0.0001010988,0.01000,0.338 26,mlp.experts.35.up_proj,0.0005657334,0.01000,0.336 26,mlp.experts.36.up_proj,0.0000812164,0.01000,0.340 26,mlp.experts.37.up_proj,0.0004163148,0.01000,0.337 26,mlp.experts.38.up_proj,0.0018171162,0.01000,0.338 26,mlp.experts.39.up_proj,0.0036170614,0.01000,0.336 26,mlp.experts.40.up_proj,0.0004380205,0.01000,0.336 26,mlp.experts.41.up_proj,0.0017677385,0.01000,0.339 26,mlp.experts.42.up_proj,0.0037455301,0.01000,0.344 26,mlp.experts.43.up_proj,0.0001899590,0.01000,0.337 26,mlp.experts.44.up_proj,0.0003814665,0.01000,0.335 26,mlp.experts.45.up_proj,0.0003594078,0.01000,0.336 26,mlp.experts.46.up_proj,0.0003301729,0.01000,0.336 26,mlp.experts.47.up_proj,0.0004232277,0.01000,0.337 26,mlp.experts.48.up_proj,0.0002107216,0.01000,0.335 26,mlp.experts.49.up_proj,0.0003444738,0.01000,0.335 26,mlp.experts.50.up_proj,0.0007518780,0.01000,0.336 26,mlp.experts.51.up_proj,0.0013061560,0.01000,0.336 26,mlp.experts.52.up_proj,0.0000833806,0.01000,0.337 26,mlp.experts.53.up_proj,0.0011209662,0.01000,0.337 26,mlp.experts.54.up_proj,0.0002885562,0.01000,0.337 26,mlp.experts.55.up_proj,0.0006038806,0.01000,0.336 26,mlp.experts.56.up_proj,0.0021688318,0.01000,0.337 26,mlp.experts.57.up_proj,0.0005900139,0.01000,0.338 26,mlp.experts.58.up_proj,0.0005082293,0.01000,0.336 26,mlp.experts.59.up_proj,0.0004724503,0.01000,0.336 26,mlp.experts.60.up_proj,0.0001463523,0.01000,0.337 26,mlp.experts.61.up_proj,0.0006531130,0.01000,0.338 26,mlp.experts.62.up_proj,0.0012811997,0.01000,0.336 26,mlp.experts.63.up_proj,0.0000975599,0.01000,0.335 26,mlp.experts.0.down_proj,0.0001042095,0.01000,0.243 26,mlp.experts.1.down_proj,0.0017479219,0.01000,0.230 26,mlp.experts.2.down_proj,0.0006493024,0.01000,0.229 26,mlp.experts.3.down_proj,0.0003051939,0.01000,0.230 26,mlp.experts.4.down_proj,0.0001983640,0.01000,0.230 26,mlp.experts.5.down_proj,0.0004668109,0.01000,0.229 26,mlp.experts.6.down_proj,0.0003464360,0.01000,0.229 26,mlp.experts.7.down_proj,0.0001624248,0.01000,0.230 26,mlp.experts.8.down_proj,0.0001672876,0.01000,0.229 26,mlp.experts.9.down_proj,0.0003153439,0.01000,0.231 26,mlp.experts.10.down_proj,0.0008005449,0.01000,0.229 26,mlp.experts.11.down_proj,0.0030500805,0.01000,0.231 26,mlp.experts.12.down_proj,0.0001691807,0.01000,0.230 26,mlp.experts.13.down_proj,0.0003248322,0.01000,0.230 26,mlp.experts.14.down_proj,0.0001135702,0.01000,0.231 26,mlp.experts.15.down_proj,0.0001054599,0.01000,0.230 26,mlp.experts.16.down_proj,0.0010923303,0.01000,0.230 26,mlp.experts.17.down_proj,0.0001200648,0.01000,0.229 26,mlp.experts.18.down_proj,0.0004900491,0.01000,0.232 26,mlp.experts.19.down_proj,0.0017993179,0.01000,0.232 26,mlp.experts.20.down_proj,0.0012904413,0.01000,0.233 26,mlp.experts.21.down_proj,0.0002112908,0.01000,0.232 26,mlp.experts.22.down_proj,0.0005577627,0.01000,0.235 26,mlp.experts.23.down_proj,0.0006276198,0.01000,0.232 26,mlp.experts.24.down_proj,0.0007475745,0.01000,0.231 26,mlp.experts.25.down_proj,0.0002320653,0.01000,0.233 26,mlp.experts.26.down_proj,0.0004627179,0.01000,0.232 26,mlp.experts.27.down_proj,0.0002715830,0.01000,0.233 26,mlp.experts.28.down_proj,0.0005187900,0.01000,0.234 26,mlp.experts.29.down_proj,0.0000981898,0.01000,0.232 26,mlp.experts.30.down_proj,0.0024850578,0.01000,0.231 26,mlp.experts.31.down_proj,0.0006485032,0.01000,0.232 26,mlp.experts.32.down_proj,0.0004073791,0.01000,0.232 26,mlp.experts.33.down_proj,0.0003270519,0.01000,0.238 26,mlp.experts.34.down_proj,0.0001115520,0.01000,0.233 26,mlp.experts.35.down_proj,0.0003622860,0.01000,0.239 26,mlp.experts.36.down_proj,0.0000541475,0.01000,0.233 26,mlp.experts.37.down_proj,0.0003015990,0.01000,0.229 26,mlp.experts.38.down_proj,0.0010781849,0.01000,0.232 26,mlp.experts.39.down_proj,0.0034362383,0.01000,0.233 26,mlp.experts.40.down_proj,0.0003518253,0.01000,0.231 26,mlp.experts.41.down_proj,0.0012898512,0.01000,0.228 26,mlp.experts.42.down_proj,0.0026332435,0.01000,0.229 26,mlp.experts.43.down_proj,0.0001546513,0.01000,0.229 26,mlp.experts.44.down_proj,0.0003178882,0.01000,0.229 26,mlp.experts.45.down_proj,0.0002528490,0.01000,0.230 26,mlp.experts.46.down_proj,0.0003086542,0.01000,0.231 26,mlp.experts.47.down_proj,0.0003117993,0.01000,0.229 26,mlp.experts.48.down_proj,0.0001283672,0.01000,0.231 26,mlp.experts.49.down_proj,0.0003160807,0.01000,0.231 26,mlp.experts.50.down_proj,0.0004215517,0.01000,0.231 26,mlp.experts.51.down_proj,0.0008587183,0.01000,0.230 26,mlp.experts.52.down_proj,0.0000510788,0.01000,0.230 26,mlp.experts.53.down_proj,0.0007267105,0.01000,0.231 26,mlp.experts.54.down_proj,0.0001822679,0.01000,0.230 26,mlp.experts.55.down_proj,0.0004223696,0.01000,0.231 26,mlp.experts.56.down_proj,0.0015177159,0.01000,0.230 26,mlp.experts.57.down_proj,0.0004004634,0.01000,0.231 26,mlp.experts.58.down_proj,0.0003022815,0.01000,0.230 26,mlp.experts.59.down_proj,0.0002948452,0.01000,0.229 26,mlp.experts.60.down_proj,0.0000802765,0.01000,0.229 26,mlp.experts.61.down_proj,0.0005014051,0.01000,0.230 26,mlp.experts.62.down_proj,0.0006548447,0.01000,0.229 26,mlp.experts.63.down_proj,0.0000890596,0.01000,0.231 26,mlp.shared_experts.gate_proj,0.0000775325,0.01000,0.346 26,mlp.shared_experts.up_proj,0.0000457248,0.01000,0.336 26,mlp.shared_experts.down_proj,0.0000071824,0.01000,0.475