{
  "config": {
    "model": "LiquidAI/LFM2.5-Encoder-350M",
    "init_from": "ProCreations/minima",
    "dataset": "HuggingFaceFW/fineweb-edu",
    "dataset_config": "sample-10BT",
    "split": "train",
    "text_column": "text",
    "output_dir": "/home/user/checkpoints/minima/encoder",
    "output_repo": null,
    "sequence_length": 1024,
    "batch_size": 8,
    "gradient_accumulation": 4,
    "steps": 16000,
    "learning_rate": 5e-05,
    "warmup_steps": 400,
    "group_size": 128,
    "recovery_rank": 0,
    "teacher_topk": 32,
    "hidden_loss_weight": 2.0,
    "layer_loss_weight": 0.5,
    "distill_loss_weight": 1.0,
    "mlm_loss_weight": 1.0,
    "temperature": 2.0,
    "mask_probability": 0.3,
    "seed": 42,
    "log_every": 10,
    "checkpoint_every": 1000,
    "resume_from": null,
    "train_ternary_weights": true,
    "max_weight_bytes": 104857600
  },
  "duration_seconds": 11887.770757198334,
  "trainable_parameters": 354483968,
  "weight_bytes": 94298568,
  "recovery_rank": 0,
  "profile": "strict",
  "history": [
    {
      "step": 1,
      "lr": 1.25e-07,
      "elapsed_s": 8.438207387924194,
      "loss": 20.382107257843018,
      "hidden": 0.8346377909183502,
      "layer": 0.3416784182190895,
      "mlm": 7.332927703857422,
      "distill": 11.209064722061157,
      "teacher_acc": 0.741391971707344
    },
    {
      "step": 11,
      "lr": 1.3750000000000002e-06,
      "elapsed_s": 15.91100549697876,
      "loss": 20.231243133544922,
      "hidden": 0.8391025960445404,
      "layer": 0.3421124964952469,
      "mlm": 7.287261247634888,
      "distill": 11.094720363616943,
      "teacher_acc": 0.7277925461530685
    },
    {
      "step": 21,
      "lr": 2.6250000000000003e-06,
      "elapsed_s": 23.412943840026855,
      "loss": 19.162318229675293,
      "hidden": 0.8315205425024033,
      "layer": 0.33662645518779755,
      "mlm": 6.99660050868988,
      "distill": 10.334363460540771,
      "teacher_acc": 0.7113983482122421
    },
    {
      "step": 31,
      "lr": 3.875e-06,
      "elapsed_s": 30.961130142211914,
      "loss": 17.63284397125244,
      "hidden": 0.8067305237054825,
      "layer": 0.3340466096997261,
      "mlm": 6.331857204437256,
      "distill": 9.52050256729126,
      "teacher_acc": 0.7246920615434647
    },
    {
      "step": 41,
      "lr": 5.125e-06,
      "elapsed_s": 38.39629864692688,
      "loss": 16.680916786193848,
      "hidden": 0.8166581839323044,
      "layer": 0.33229923993349075,
      "mlm": 5.905057072639465,
      "distill": 8.976394414901733,
      "teacher_acc": 0.7295158505439758
    },
    {
      "step": 51,
      "lr": 6.375000000000001e-06,
      "elapsed_s": 45.90352153778076,
      "loss": 15.108635425567627,
      "hidden": 0.8116884082555771,
      "layer": 0.3292075991630554,
      "mlm": 5.3271849155426025,
      "distill": 7.993469953536987,
      "teacher_acc": 0.720592126250267
    },
    {
      "step": 61,
      "lr": 7.625000000000001e-06,
      "elapsed_s": 53.383700370788574,
      "loss": 14.412956714630127,
      "hidden": 0.7955577671527863,
      "layer": 0.32568876445293427,
      "mlm": 4.998058438301086,
      "distill": 7.660938382148743,
      "teacher_acc": 0.731194406747818
    },
    {
      "step": 71,
      "lr": 8.875e-06,
      "elapsed_s": 60.770543336868286,
      "loss": 13.60209035873413,
      "hidden": 0.7885428071022034,
      "layer": 0.316788449883461,
      "mlm": 4.6275060176849365,
      "distill": 7.239104628562927,
      "teacher_acc": 0.728519082069397
    },
    {
      "step": 81,
      "lr": 1.0124999999999999e-05,
      "elapsed_s": 68.12384033203125,
      "loss": 13.014343976974487,
      "hidden": 0.7739235609769821,
      "layer": 0.31509388238191605,
      "mlm": 4.396705627441406,
      "distill": 6.912244200706482,
      "teacher_acc": 0.7272239178419113
    },
    {
      "step": 91,
      "lr": 1.1375e-05,
      "elapsed_s": 75.48223567008972,
      "loss": 12.397578477859497,
      "hidden": 0.7607980370521545,
      "layer": 0.31272996962070465,
      "mlm": 4.1447343826293945,
      "distill": 6.574883222579956,
      "teacher_acc": 0.7345134615898132
    },
    {
      "step": 101,
      "lr": 1.2625000000000002e-05,
      "elapsed_s": 82.84921288490295,
      "loss": 12.093305349349976,
      "hidden": 0.7483604401350021,
      "layer": 0.3083856627345085,
      "mlm": 4.03705632686615,
      "distill": 6.405335307121277,
      "teacher_acc": 0.7179478406906128
    },
    {
      "step": 111,
      "lr": 1.3875e-05,
      "elapsed_s": 90.21919751167297,
      "loss": 11.945655822753906,
      "hidden": 0.7515987753868103,
      "layer": 0.30460746586322784,
      "mlm": 3.9536109566688538,
      "distill": 6.336543560028076,
      "teacher_acc": 0.7208492904901505
    },
    {
      "step": 121,
      "lr": 1.5125000000000002e-05,
      "elapsed_s": 97.58434200286865,
      "loss": 11.234977960586548,
      "hidden": 0.7261582016944885,
      "layer": 0.2982270270586014,
      "mlm": 3.7661255598068237,
      "distill": 5.867422580718994,
      "teacher_acc": 0.7092819511890411
    },
    {
      "step": 131,
      "lr": 1.6375e-05,
      "elapsed_s": 104.9761278629303,
      "loss": 11.587674856185913,
      "hidden": 0.7305631786584854,
      "layer": 0.2971068322658539,
      "mlm": 3.7347702383995056,
      "distill": 6.2432249784469604,
      "teacher_acc": 0.7257865965366364
    },
    {
      "step": 141,
      "lr": 1.7625e-05,
      "elapsed_s": 112.3357400894165,
      "loss": 10.880138397216797,
      "hidden": 0.6972691863775253,
      "layer": 0.2890807241201401,
      "mlm": 3.661649167537689,
      "distill": 5.679410457611084,
      "teacher_acc": 0.7014146000146866
    },
    {
      "step": 151,
      "lr": 1.8875e-05,
      "elapsed_s": 119.68427348136902,
      "loss": 11.062957286834717,
      "hidden": 0.6857605576515198,
      "layer": 0.28529219329357147,
      "mlm": 3.5269588232040405,
      "distill": 6.021831154823303,
      "teacher_acc": 0.7346127033233643
    },
    {
      "step": 161,
      "lr": 2.0125e-05,
      "elapsed_s": 127.03348422050476,
      "loss": 11.272920608520508,
      "hidden": 0.7007472217082977,
      "layer": 0.2829109951853752,
      "mlm": 3.5320947766304016,
      "distill": 6.1978759765625,
      "teacher_acc": 0.7471081763505936
    },
    {
      "step": 171,
      "lr": 2.1375e-05,
      "elapsed_s": 134.39313292503357,
      "loss": 10.751477241516113,
      "hidden": 0.6778112649917603,
      "layer": 0.2782456651329994,
      "mlm": 3.4939733743667603,
      "distill": 5.762758493423462,
      "teacher_acc": 0.7287603914737701
    },
    {
      "step": 181,
      "lr": 2.2625e-05,
      "elapsed_s": 141.71666169166565,
      "loss": 10.61971139907837,
      "hidden": 0.6646811664104462,
      "layer": 0.27351267635822296,
      "mlm": 3.4403656721115112,
      "distill": 5.713227152824402,
      "teacher_acc": 0.7304104864597321
    },
    {
      "step": 191,
      "lr": 2.3875000000000004e-05,
      "elapsed_s": 149.05428099632263,
      "loss": 10.290287256240845,
      "hidden": 0.6457851976156235,
      "layer": 0.26799193024635315,
      "mlm": 3.395330488681793,
      "distill": 5.469390392303467,
      "teacher_acc": 0.7201550453901291
    },
    {
      "step": 201,
      "lr": 2.5125e-05,
      "elapsed_s": 156.44250774383545,
      "loss": 10.08063817024231,
      "hidden": 0.6336404383182526,
      "layer": 0.2623717039823532,
      "mlm": 3.344578802585602,
      "distill": 5.337592363357544,
      "teacher_acc": 0.7108413875102997
    },
    {
      "step": 211,
      "lr": 2.6375e-05,
      "elapsed_s": 163.8493993282318,
      "loss": 9.99802565574646,
      "hidden": 0.6193995177745819,
      "layer": 0.25645096600055695,
      "mlm": 3.2771191596984863,
      "distill": 5.353882193565369,
      "teacher_acc": 0.7279073297977448
    },
    {
      "step": 221,
      "lr": 2.7625e-05,
      "elapsed_s": 171.25341939926147,
      "loss": 9.680199146270752,
      "hidden": 0.6044565886259079,
      "layer": 0.2507324330508709,
      "mlm": 3.2872328758239746,
      "distill": 5.05868661403656,
      "teacher_acc": 0.7153979390859604
    },
    {
      "step": 231,
      "lr": 2.8875000000000004e-05,
      "elapsed_s": 178.6449475288391,
      "loss": 9.647573709487915,
      "hidden": 0.5889715254306793,
      "layer": 0.24639014899730682,
      "mlm": 3.243684768676758,
      "distill": 5.102750778198242,
      "teacher_acc": 0.7167518138885498
    },
    {
      "step": 241,
      "lr": 3.0125e-05,
      "elapsed_s": 186.02465295791626,
      "loss": 9.525534629821777,
      "hidden": 0.5796572864055634,
      "layer": 0.2429041862487793,
      "mlm": 3.2263808250427246,
      "distill": 5.018387198448181,
      "teacher_acc": 0.7235804498195648
    },
    {
      "step": 251,
      "lr": 3.1375000000000004e-05,
      "elapsed_s": 193.4231698513031,
      "loss": 9.560251474380493,
      "hidden": 0.57901830971241,
      "layer": 0.24033616855740547,
      "mlm": 3.174620270729065,
      "distill": 5.107426285743713,
      "teacher_acc": 0.7383224368095398
    },
    {
      "step": 261,
      "lr": 3.2625e-05,
      "elapsed_s": 200.81871700286865,
      "loss": 8.957597255706787,
      "hidden": 0.5617494434118271,
      "layer": 0.23227909952402115,
      "mlm": 3.145342469215393,
      "distill": 4.572616219520569,
      "teacher_acc": 0.715526819229126
    },
    {
      "step": 271,
      "lr": 3.3875000000000003e-05,
      "elapsed_s": 208.20436096191406,
      "loss": 9.00001859664917,
      "hidden": 0.5483827590942383,
      "layer": 0.2310510240495205,
      "mlm": 3.1237074732780457,
      "distill": 4.66402006149292,
      "teacher_acc": 0.7117209136486053
    },
    {
      "step": 281,
      "lr": 3.5125e-05,
      "elapsed_s": 215.57980751991272,
      "loss": 8.769193887710571,
      "hidden": 0.5332635194063187,
      "layer": 0.22395243868231773,
      "mlm": 3.0474495887756348,
      "distill": 4.5432411432266235,
      "teacher_acc": 0.7251143306493759
    },
    {
      "step": 291,
      "lr": 3.6375e-05,
      "elapsed_s": 222.9703459739685,
      "loss": 8.737284183502197,
      "hidden": 0.5418843030929565,
      "layer": 0.2234797701239586,
      "mlm": 3.039504826068878,
      "distill": 4.502270817756653,
      "teacher_acc": 0.7075243443250656
    },
    {
      "step": 301,
      "lr": 3.7625e-05,
      "elapsed_s": 230.36128664016724,
      "loss": 8.394862651824951,
      "hidden": 0.5147878527641296,
      "layer": 0.21833140030503273,
      "mlm": 2.999567449092865,
      "distill": 4.256553769111633,
      "teacher_acc": 0.7175053507089615
    },
    {
      "step": 311,
      "lr": 3.8875e-05,
      "elapsed_s": 237.73944878578186,
      "loss": 8.387603521347046,
      "hidden": 0.505688264966011,
      "layer": 0.21406494081020355,
      "mlm": 2.982984960079193,
      "distill": 4.286209940910339,
      "teacher_acc": 0.7223232388496399
    },
    {
      "step": 321,
      "lr": 4.0125000000000006e-05,
      "elapsed_s": 245.1364848613739,
      "loss": 8.304641962051392,
      "hidden": 0.5054546743631363,
      "layer": 0.2127266526222229,
      "mlm": 2.9435108304023743,
      "distill": 4.243858635425568,
      "teacher_acc": 0.7009203433990479
    },
    {
      "step": 331,
      "lr": 4.1375e-05,
      "elapsed_s": 252.5275890827179,
      "loss": 7.921196460723877,
      "hidden": 0.49171698838472366,
      "layer": 0.20450466871261597,
      "mlm": 2.8488515615463257,
      "distill": 3.9866586327552795,
      "teacher_acc": 0.7052172422409058
    },
    {
      "step": 341,
      "lr": 4.2625e-05,
      "elapsed_s": 259.9027352333069,
      "loss": 7.768313050270081,
      "hidden": 0.48448561131954193,
      "layer": 0.20270055532455444,
      "mlm": 2.870379626750946,
      "distill": 3.8276119232177734,
      "teacher_acc": 0.7051264345645905
    },
    {
      "step": 351,
      "lr": 4.3874999999999996e-05,
      "elapsed_s": 267.29892110824585,
      "loss": 7.813935875892639,
      "hidden": 0.4834267273545265,
      "layer": 0.1991100274026394,
      "mlm": 2.8287346959114075,
      "distill": 3.9187928438186646,
      "teacher_acc": 0.717182919383049
    },
    {
      "step": 361,
      "lr": 4.5125e-05,
      "elapsed_s": 274.69245076179504,
      "loss": 8.06712520122528,
      "hidden": 0.48718203604221344,
      "layer": 0.2007831148803234,
      "mlm": 2.7557015419006348,
      "distill": 4.236667990684509,
      "teacher_acc": 0.7378624975681305
    },
    {
      "step": 371,
      "lr": 4.6375e-05,
      "elapsed_s": 282.12883591651917,
      "loss": 7.616874098777771,
      "hidden": 0.4762837290763855,
      "layer": 0.19511355832219124,
      "mlm": 2.702797293663025,
      "distill": 3.8639525175094604,
      "teacher_acc": 0.7345853894948959
    },
    {
      "step": 381,
      "lr": 4.7625000000000006e-05,
      "elapsed_s": 289.5580132007599,
      "loss": 7.545713305473328,
      "hidden": 0.4575069919228554,
      "layer": 0.19037995114922523,
      "mlm": 2.7477957606315613,
      "distill": 3.7877135276794434,
      "teacher_acc": 0.7281907051801682
    },
    {
      "step": 391,
      "lr": 4.8875e-05,
      "elapsed_s": 296.96427512168884,
      "loss": 7.493480682373047,
      "hidden": 0.4638398289680481,
      "layer": 0.1900879219174385,
      "mlm": 2.737092614173889,
      "distill": 3.7336645126342773,
      "teacher_acc": 0.7123801559209824
    },
    {
      "step": 401,
      "lr": 5e-05,
      "elapsed_s": 304.3294451236725,
      "loss": 6.972379684448242,
      "hidden": 0.4409460425376892,
      "layer": 0.18277034163475037,
      "mlm": 2.6389081478118896,
      "distill": 3.360194206237793,
      "teacher_acc": 0.7027393281459808
    },
    {
      "step": 411,
      "lr": 4.999994930555009e-05,
      "elapsed_s": 311.6836335659027,
      "loss": 7.326171636581421,
      "hidden": 0.4544595927000046,
      "layer": 0.1839304305613041,
      "mlm": 2.5980085730552673,
      "distill": 3.727278709411621,
      "teacher_acc": 0.7458912879228592
    },
    {
      "step": 421,
      "lr": 4.999979722240593e-05,
      "elapsed_s": 319.09366035461426,
      "loss": 7.155426383018494,
      "hidden": 0.4476668983697891,
      "layer": 0.18222976848483086,
      "mlm": 2.6186808943748474,
      "distill": 3.550296723842621,
      "teacher_acc": 0.7102632969617844
    },
    {
      "step": 431,
      "lr": 4.999954375118431e-05,
      "elapsed_s": 326.4686439037323,
      "loss": 7.080630898475647,
      "hidden": 0.435101181268692,
      "layer": 0.1784873567521572,
      "mlm": 2.5806180238723755,
      "distill": 3.5405668020248413,
      "teacher_acc": 0.7217626422643661
    },
    {
      "step": 441,
      "lr": 4.99991888929132e-05,
      "elapsed_s": 333.85961842536926,
      "loss": 6.914414763450623,
      "hidden": 0.4366634264588356,
      "layer": 0.17913294956088066,
      "mlm": 2.5914214849472046,
      "distill": 3.3600999116897583,
      "teacher_acc": 0.7197299003601074
    },
    {
      "step": 451,
      "lr": 4.9998732649031746e-05,
      "elapsed_s": 341.27760314941406,
      "loss": 6.717426776885986,
      "hidden": 0.4241471514105797,
      "layer": 0.1744411140680313,
      "mlm": 2.5557414889335632,
      "distill": 3.2261704206466675,
      "teacher_acc": 0.7188067734241486
    },
    {
      "step": 461,
      "lr": 4.999817502139027e-05,
      "elapsed_s": 348.68601083755493,
      "loss": 6.747365236282349,
      "hidden": 0.426373228430748,
      "layer": 0.1708102747797966,
      "mlm": 2.4684707522392273,
      "distill": 3.3407429456710815,
      "teacher_acc": 0.7260115593671799
    },
    {
      "step": 471,
      "lr": 4.999751601225026e-05,
      "elapsed_s": 356.1031880378723,
      "loss": 6.7500081062316895,
      "hidden": 0.42207154631614685,
      "layer": 0.17049676924943924,
      "mlm": 2.4504913091659546,
      "distill": 3.3701255321502686,
      "teacher_acc": 0.7383766919374466
    },
    {
      "step": 481,
      "lr": 4.999675562428437e-05,
      "elapsed_s": 363.4732165336609,
      "loss": 6.7328526973724365,
      "hidden": 0.41543419659137726,
      "layer": 0.16748036071658134,
      "mlm": 2.4592020511627197,
      "distill": 3.3590421080589294,
      "teacher_acc": 0.731430396437645
    },
    {
      "step": 491,
      "lr": 4.9995893860576395e-05,
      "elapsed_s": 370.86935925483704,
      "loss": 6.760748267173767,
      "hidden": 0.4206240624189377,
      "layer": 0.16873305663466454,
      "mlm": 2.527904212474823,
      "distill": 3.307229459285736,
      "teacher_acc": 0.7193769365549088
    },
    {
      "step": 501,
      "lr": 4.999493072462126e-05,
      "elapsed_s": 378.2529573440552,
      "loss": 6.353148698806763,
      "hidden": 0.40859969705343246,
      "layer": 0.1644362322986126,
      "mlm": 2.585409462451935,
      "distill": 2.868321716785431,
      "teacher_acc": 0.6887723803520203
    },
    {
      "step": 511,
      "lr": 4.9993866220325016e-05,
      "elapsed_s": 385.65176367759705,
      "loss": 6.216914534568787,
      "hidden": 0.40293149650096893,
      "layer": 0.1617247387766838,
      "mlm": 2.4225248098373413,
      "distill": 2.9076645374298096,
      "teacher_acc": 0.7090970426797867
    },
    {
      "step": 521,
      "lr": 4.999270035200483e-05,
      "elapsed_s": 393.05822372436523,
      "loss": 6.242516756057739,
      "hidden": 0.4046694040298462,
      "layer": 0.16077996790409088,
      "mlm": 2.341669797897339,
      "distill": 3.0111181139945984,
      "teacher_acc": 0.7241754531860352
    },
    {
      "step": 531,
      "lr": 4.999143312438893e-05,
      "elapsed_s": 400.46572065353394,
      "loss": 6.31710147857666,
      "hidden": 0.3994235023856163,
      "layer": 0.16054637730121613,
      "mlm": 2.462680220603943,
      "distill": 2.9753010869026184,
      "teacher_acc": 0.7089099436998367
    },
    {
      "step": 541,
      "lr": 4.9990064542616654e-05,
      "elapsed_s": 407.88762855529785,
      "loss": 5.981846332550049,
      "hidden": 0.3858133405447006,
      "layer": 0.15531393885612488,
      "mlm": 2.3307279348373413,
      "distill": 2.8018347024917603,
      "teacher_acc": 0.7229381501674652
    },
    {
      "step": 551,
      "lr": 4.9988594612238336e-05,
      "elapsed_s": 415.29609417915344,
      "loss": 6.248854994773865,
      "hidden": 0.3978895992040634,
      "layer": 0.15689504891633987,
      "mlm": 2.418695569038391,
      "distill": 2.9559327960014343,
      "teacher_acc": 0.7165200561285019
    },
    {
      "step": 561,
      "lr": 4.9987023339215374e-05,
      "elapsed_s": 422.70060658454895,
      "loss": 6.041122794151306,
      "hidden": 0.3875919282436371,
      "layer": 0.15600741654634476,
      "mlm": 2.3161149621009827,
      "distill": 2.8718201518058777,
      "teacher_acc": 0.7301480919122696
    },
    {
      "step": 571,
      "lr": 4.9985350729920153e-05,
      "elapsed_s": 430.1006269454956,
      "loss": 5.758399605751038,
      "hidden": 0.37757109105587006,
      "layer": 0.1502341777086258,
      "mlm": 2.241311192512512,
      "distill": 2.686829090118408,
      "teacher_acc": 0.7183603644371033
    },
    {
      "step": 581,
      "lr": 4.998357679113603e-05,
      "elapsed_s": 437.50221991539,
      "loss": 5.868234634399414,
      "hidden": 0.38468971848487854,
      "layer": 0.1529390700161457,
      "mlm": 2.305842936038971,
      "distill": 2.716542661190033,
      "teacher_acc": 0.7171104699373245
    },
    {
      "step": 591,
      "lr": 4.998170153005732e-05,
      "elapsed_s": 444.8764066696167,
      "loss": 5.976639986038208,
      "hidden": 0.39913227409124374,
      "layer": 0.15395262837409973,
      "mlm": 2.2142714262008667,
      "distill": 2.887127697467804,
      "teacher_acc": 0.7405375242233276
    },
    {
      "step": 601,
      "lr": 4.9979724954289244e-05,
      "elapsed_s": 452.2580897808075,
      "loss": 5.784687042236328,
      "hidden": 0.3792489990592003,
      "layer": 0.14960836619138718,
      "mlm": 2.266518771648407,
      "distill": 2.684866189956665,
      "teacher_acc": 0.7305253446102142
    },
    {
      "step": 611,
      "lr": 4.997764707184791e-05,
      "elapsed_s": 459.6414020061493,
      "loss": 5.6485384702682495,
      "hidden": 0.37860359996557236,
      "layer": 0.14826326817274094,
      "mlm": 2.1599996089935303,
      "distill": 2.65719997882843,
      "teacher_acc": 0.734072357416153
    },
    {
      "step": 621,
      "lr": 4.99754678911603e-05,
      "elapsed_s": 467.03337574005127,
      "loss": 5.639063119888306,
      "hidden": 0.37454646825790405,
      "layer": 0.14888331294059753,
      "mlm": 2.3160704374313354,
      "distill": 2.4994580149650574,
      "teacher_acc": 0.7011914700269699
    },
    {
      "step": 631,
      "lr": 4.9973187421064194e-05,
      "elapsed_s": 474.40744400024414,
      "loss": 5.670869708061218,
      "hidden": 0.3767525181174278,
      "layer": 0.14850180223584175,
      "mlm": 2.145396411418915,
      "distill": 2.6977174282073975,
      "teacher_acc": 0.7550195753574371
    },
    {
      "step": 641,
      "lr": 4.997080567080817e-05,
      "elapsed_s": 481.8111844062805,
      "loss": 5.583534002304077,
      "hidden": 0.37513643503189087,
      "layer": 0.14605392143130302,
      "mlm": 2.23726749420166,
      "distill": 2.5229666233062744,
      "teacher_acc": 0.7129390090703964
    },
    {
      "step": 651,
      "lr": 4.996832265005154e-05,
      "elapsed_s": 489.18649673461914,
      "loss": 5.6264495849609375,
      "hidden": 0.3813329115509987,
      "layer": 0.14795170724391937,
      "mlm": 2.212577164173126,
      "distill": 2.57723069190979,
      "teacher_acc": 0.7245818227529526
    },
    {
      "step": 661,
      "lr": 4.996573836886435e-05,
      "elapsed_s": 496.55889439582825,
      "loss": 5.444245219230652,
      "hidden": 0.3595373034477234,
      "layer": 0.14399639889597893,
      "mlm": 2.158246397972107,
      "distill": 2.49492609500885,
      "teacher_acc": 0.7313836961984634
    },
    {
      "step": 671,
      "lr": 4.9963052837727286e-05,
      "elapsed_s": 503.98105096817017,
      "loss": 5.625387787818909,
      "hidden": 0.3801627680659294,
      "layer": 0.1467006802558899,
      "mlm": 2.1434438228607178,
      "distill": 2.648267984390259,
      "teacher_acc": 0.7389213591814041
    },
    {
      "step": 681,
      "lr": 4.996026606753167e-05,
      "elapsed_s": 511.39908933639526,
      "loss": 5.253180503845215,
      "hidden": 0.35616493970155716,
      "layer": 0.14314910024404526,
      "mlm": 2.14260071516037,
      "distill": 2.326675236225128,
      "teacher_acc": 0.7218354195356369
    },
    {
      "step": 691,
      "lr": 4.995737806957941e-05,
      "elapsed_s": 518.812028169632,
      "loss": 5.235458612442017,
      "hidden": 0.34271908551454544,
      "layer": 0.14079836010932922,
      "mlm": 2.139342725276947,
      "distill": 2.340278446674347,
      "teacher_acc": 0.7233784198760986
    },
    {
      "step": 701,
      "lr": 4.995438885558294e-05,
      "elapsed_s": 526.2391893863678,
      "loss": 5.291362166404724,
      "hidden": 0.35482554882764816,
      "layer": 0.1406991109251976,
      "mlm": 2.1557281613349915,
      "distill": 2.3556333780288696,
      "teacher_acc": 0.724240243434906
    },
    {
      "step": 711,
      "lr": 4.995129843766518e-05,
      "elapsed_s": 533.6620616912842,
      "loss": 5.245983600616455,
      "hidden": 0.36287378519773483,
      "layer": 0.14272823557257652,
      "mlm": 2.1766187846660614,
      "distill": 2.2722529768943787,
      "teacher_acc": 0.7111829370260239
    },
    {
      "step": 721,
      "lr": 4.994810682835951e-05,
      "elapsed_s": 541.1082148551941,
      "loss": 5.072383642196655,
      "hidden": 0.3412201628088951,
      "layer": 0.13884487748146057,
      "mlm": 2.0981135964393616,
      "distill": 2.222407341003418,
      "teacher_acc": 0.720179557800293
    },
    {
      "step": 731,
      "lr": 4.9944814040609655e-05,
      "elapsed_s": 548.5335462093353,
      "loss": 5.15772557258606,
      "hidden": 0.3424356132745743,
      "layer": 0.13967958837747574,
      "mlm": 2.1708444356918335,
      "distill": 2.2321701049804688,
      "teacher_acc": 0.7167054563760757
    },
    {
      "step": 741,
      "lr": 4.994142008776972e-05,
      "elapsed_s": 555.9521386623383,
      "loss": 5.285245299339294,
      "hidden": 0.35129498690366745,
      "layer": 0.1415005959570408,
      "mlm": 2.142484486103058,
      "distill": 2.3694204688072205,
      "teacher_acc": 0.7157605886459351
    },
    {
      "step": 751,
      "lr": 4.993792498360407e-05,
      "elapsed_s": 563.3601932525635,
      "loss": 5.073930501937866,
      "hidden": 0.3416658937931061,
      "layer": 0.13778120651841164,
      "mlm": 2.0428466498851776,
      "distill": 2.278861403465271,
      "teacher_acc": 0.736793115735054
    },
    {
      "step": 761,
      "lr": 4.9934328742287285e-05,
      "elapsed_s": 570.8088579177856,
      "loss": 5.146379232406616,
      "hidden": 0.3481604531407356,
      "layer": 0.13948576524853706,
      "mlm": 2.0327190160751343,
      "distill": 2.347596526145935,
      "teacher_acc": 0.7375090420246124
    },
    {
      "step": 771,
      "lr": 4.9930631378404126e-05,
      "elapsed_s": 578.2377731800079,
      "loss": 5.095861792564392,
      "hidden": 0.34284236282110214,
      "layer": 0.1386152170598507,
      "mlm": 2.0887444019317627,
      "distill": 2.2521252036094666,
      "teacher_acc": 0.723158210515976
    },
    {
      "step": 781,
      "lr": 4.992683290694946e-05,
      "elapsed_s": 585.661413192749,
      "loss": 5.174114227294922,
      "hidden": 0.3386623188853264,
      "layer": 0.13830074667930603,
      "mlm": 2.135282278060913,
      "distill": 2.29235702753067,
      "teacher_acc": 0.7139520049095154
    },
    {
      "step": 791,
      "lr": 4.99229333433282e-05,
      "elapsed_s": 593.0888574123383,
      "loss": 4.989296197891235,
      "hidden": 0.3369723930954933,
      "layer": 0.13672567531466484,
      "mlm": 2.0412542819976807,
      "distill": 2.2057341933250427,
      "teacher_acc": 0.7327211797237396
    },
    {
      "step": 801,
      "lr": 4.9918932703355256e-05,
      "elapsed_s": 600.5246045589447,
      "loss": 4.930473446846008,
      "hidden": 0.3342827931046486,
      "layer": 0.13438121229410172,
      "mlm": 2.088502526283264,
      "distill": 2.1062145829200745,
      "teacher_acc": 0.7142133265733719
    },
    {
      "step": 811,
      "lr": 4.991483100325542e-05,
      "elapsed_s": 607.9515490531921,
      "loss": 5.094535708427429,
      "hidden": 0.34925705194473267,
      "layer": 0.1373111829161644,
      "mlm": 2.0720496773719788,
      "distill": 2.2553164958953857,
      "teacher_acc": 0.7279532998800278
    },
    {
      "step": 821,
      "lr": 4.9910628259663404e-05,
      "elapsed_s": 615.3874452114105,
      "loss": 4.867947697639465,
      "hidden": 0.3341575264930725,
      "layer": 0.13361690938472748,
      "mlm": 2.0251238346099854,
      "distill": 2.107700228691101,
      "teacher_acc": 0.7225870937108994
    },
    {
      "step": 831,
      "lr": 4.990632448962365e-05,
      "elapsed_s": 622.805579662323,
      "loss": 5.057167053222656,
      "hidden": 0.33050037175416946,
      "layer": 0.1347789242863655,
      "mlm": 2.109598398208618,
      "distill": 2.2191783785820007,
      "teacher_acc": 0.7204168140888214
    },
    {
      "step": 841,
      "lr": 4.990191971059033e-05,
      "elapsed_s": 630.2259871959686,
      "loss": 5.001285433769226,
      "hidden": 0.3280780613422394,
      "layer": 0.1360291987657547,
      "mlm": 2.083369165658951,
      "distill": 2.193745493888855,
      "teacher_acc": 0.7262595146894455
    },
    {
      "step": 851,
      "lr": 4.989741394042727e-05,
      "elapsed_s": 637.64075922966,
      "loss": 5.136803388595581,
      "hidden": 0.3375082463026047,
      "layer": 0.1416374370455742,
      "mlm": 2.142011284828186,
      "distill": 2.2489569187164307,
      "teacher_acc": 0.7158180624246597
    },
    {
      "step": 861,
      "lr": 4.98928071974079e-05,
      "elapsed_s": 645.0270721912384,
      "loss": 5.104859948158264,
      "hidden": 0.3423059210181236,
      "layer": 0.1369442194700241,
      "mlm": 2.0107484459877014,
      "distill": 2.3410274982452393,
      "teacher_acc": 0.7325365543365479
    },
    {
      "step": 871,
      "lr": 4.988809950021511e-05,
      "elapsed_s": 652.4525923728943,
      "loss": 4.788349509239197,
      "hidden": 0.32434888929128647,
      "layer": 0.13105837628245354,
      "mlm": 1.9612310826778412,
      "distill": 2.112891435623169,
      "teacher_acc": 0.7467294037342072
    },
    {
      "step": 881,
      "lr": 4.988329086794122e-05,
      "elapsed_s": 659.8837642669678,
      "loss": 5.037511944770813,
      "hidden": 0.3343901112675667,
      "layer": 0.14045323058962822,
      "mlm": 2.0418640077114105,
      "distill": 2.256641149520874,
      "teacher_acc": 0.7328261435031891
    },
    {
      "step": 891,
      "lr": 4.987838132008791e-05,
      "elapsed_s": 667.2961189746857,
      "loss": 4.8361616134643555,
      "hidden": 0.3259722888469696,
      "layer": 0.1341593973338604,
      "mlm": 2.0181438326835632,
      "distill": 2.0989935100078583,
      "teacher_acc": 0.7223280966281891
    },
    {
      "step": 901,
      "lr": 4.987337087656614e-05,
      "elapsed_s": 674.7099261283875,
      "loss": 4.822410941123962,
      "hidden": 0.32974011451005936,
      "layer": 0.13251709565520287,
      "mlm": 2.0140772461891174,
      "distill": 2.082594931125641,
      "teacher_acc": 0.7189608961343765
    },
    {
      "step": 911,
      "lr": 4.986825955769604e-05,
      "elapsed_s": 682.1485304832458,
      "loss": 4.8774436712265015,
      "hidden": 0.33215662091970444,
      "layer": 0.13729573786258698,
      "mlm": 2.0375206768512726,
      "distill": 2.106961876153946,
      "teacher_acc": 0.7254447191953659
    },
    {
      "step": 921,
      "lr": 4.9863047384206835e-05,
      "elapsed_s": 689.5860855579376,
      "loss": 4.879358887672424,
      "hidden": 0.33479148894548416,
      "layer": 0.13709333539009094,
      "mlm": 2.0246306359767914,
      "distill": 2.1165987253189087,
      "teacher_acc": 0.7159988582134247
    },
    {
      "step": 931,
      "lr": 4.98577343772368e-05,
      "elapsed_s": 696.9890751838684,
      "loss": 4.817743897438049,
      "hidden": 0.3220086693763733,
      "layer": 0.13513080030679703,
      "mlm": 2.078570246696472,
      "distill": 2.0275908410549164,
      "teacher_acc": 0.6991146951913834
    },
    {
      "step": 941,
      "lr": 4.985232055833313e-05,
      "elapsed_s": 704.3758549690247,
      "loss": 4.817591190338135,
      "hidden": 0.32111839950084686,
      "layer": 0.13454684615135193,
      "mlm": 2.0539687871932983,
      "distill": 2.0541121065616608,
      "teacher_acc": 0.7147301286458969
    },
    {
      "step": 951,
      "lr": 4.984680594945187e-05,
      "elapsed_s": 711.804812669754,
      "loss": 4.800841569900513,
      "hidden": 0.3201806917786598,
      "layer": 0.1298685036599636,
      "mlm": 2.092036545276642,
      "distill": 2.0035094618797302,
      "teacher_acc": 0.6876754313707352
    },
    {
      "step": 961,
      "lr": 4.984119057295783e-05,
      "elapsed_s": 719.2115852832794,
      "loss": 4.76696240901947,
      "hidden": 0.3226161077618599,
      "layer": 0.13571170717477798,
      "mlm": 1.979168713092804,
      "distill": 2.0747055113315582,
      "teacher_acc": 0.7146732956171036
    },
    {
      "step": 971,
      "lr": 4.983547445162447e-05,
      "elapsed_s": 726.6001932621002,
      "loss": 4.897698402404785,
      "hidden": 0.3398028537631035,
      "layer": 0.1372847780585289,
      "mlm": 1.9599759578704834,
      "distill": 2.1894743740558624,
      "teacher_acc": 0.7334188222885132
    },
    {
      "step": 981,
      "lr": 4.982965760863385e-05,
      "elapsed_s": 734.017664194107,
      "loss": 4.791213870048523,
      "hidden": 0.3227470964193344,
      "layer": 0.13101212307810783,
      "mlm": 2.0096512734889984,
      "distill": 2.070562422275543,
      "teacher_acc": 0.7185997366905212
    },
    {
      "step": 991,
      "lr": 4.982374006757651e-05,
      "elapsed_s": 741.4150278568268,
      "loss": 4.641362905502319,
      "hidden": 0.3129960596561432,
      "layer": 0.12909220345318317,
      "mlm": 1.9589324593544006,
      "distill": 1.9918922185897827,
      "teacher_acc": 0.7298399060964584
    },
    {
      "step": 1001,
      "lr": 4.981772185245135e-05,
      "elapsed_s": 750.3298959732056,
      "loss": 4.703034400939941,
      "hidden": 0.3219163417816162,
      "layer": 0.13305561244487762,
      "mlm": 1.922544777393341,
      "distill": 2.0701292753219604,
      "teacher_acc": 0.7312430292367935
    },
    {
      "step": 1011,
      "lr": 4.98116029876656e-05,
      "elapsed_s": 757.7499992847443,
      "loss": 4.610443115234375,
      "hidden": 0.3101794049143791,
      "layer": 0.1295454427599907,
      "mlm": 1.9870969653129578,
      "distill": 1.938214510679245,
      "teacher_acc": 0.7150159627199173
    },
    {
      "step": 1021,
      "lr": 4.9805383498034634e-05,
      "elapsed_s": 765.1673581600189,
      "loss": 4.622528553009033,
      "hidden": 0.3260825648903847,
      "layer": 0.1337474249303341,
      "mlm": 1.8741153180599213,
      "distill": 2.0293743014335632,
      "teacher_acc": 0.7508395463228226
    },
    {
      "step": 1031,
      "lr": 4.9799063408781964e-05,
      "elapsed_s": 772.5650100708008,
      "loss": 4.572354793548584,
      "hidden": 0.3046359047293663,
      "layer": 0.12889032438397408,
      "mlm": 1.9670059084892273,
      "distill": 1.9316319525241852,
      "teacher_acc": 0.7208455950021744
    },
    {
      "step": 1041,
      "lr": 4.979264274553905e-05,
      "elapsed_s": 779.9983477592468,
      "loss": 4.474694609642029,
      "hidden": 0.30292869359254837,
      "layer": 0.12567608430981636,
      "mlm": 1.8532128632068634,
      "distill": 1.952786237001419,
      "teacher_acc": 0.74128757417202
    },
    {
      "step": 1051,
      "lr": 4.9786121534345265e-05,
      "elapsed_s": 787.3985753059387,
      "loss": 4.554490923881531,
      "hidden": 0.3086903840303421,
      "layer": 0.12875917926430702,
      "mlm": 1.9715653657913208,
      "distill": 1.9011651575565338,
      "teacher_acc": 0.7115155607461929
    },
    {
      "step": 1061,
      "lr": 4.977949980164773e-05,
      "elapsed_s": 794.7527594566345,
      "loss": 4.721103668212891,
      "hidden": 0.311616025865078,
      "layer": 0.13088103383779526,
      "mlm": 2.0440208315849304,
      "distill": 1.9884102046489716,
      "teacher_acc": 0.7057228833436966
    },
    {
      "step": 1071,
      "lr": 4.977277757430126e-05,
      "elapsed_s": 802.1160862445831,
      "loss": 4.518136620521545,
      "hidden": 0.312306210398674,
      "layer": 0.12948568351566792,
      "mlm": 1.9269156754016876,
      "distill": 1.9018657207489014,
      "teacher_acc": 0.7256583422422409
    },
    {
      "step": 1081,
      "lr": 4.976595487956823e-05,
      "elapsed_s": 809.4879651069641,
      "loss": 4.508713006973267,
      "hidden": 0.3133067935705185,
      "layer": 0.13071466237306595,
      "mlm": 1.8745789229869843,
      "distill": 1.942163109779358,
      "teacher_acc": 0.7400075644254684
    },
    {
      "step": 1091,
      "lr": 4.975903174511846e-05,
      "elapsed_s": 816.9056124687195,
      "loss": 4.672260165214539,
      "hidden": 0.30830634385347366,
      "layer": 0.13050559163093567,
      "mlm": 2.0098283886909485,
      "distill": 1.9805662333965302,
      "teacher_acc": 0.7097439765930176
    },
    {
      "step": 1101,
      "lr": 4.97520081990291e-05,
      "elapsed_s": 824.3218035697937,
      "loss": 4.478162169456482,
      "hidden": 0.30132005363702774,
      "layer": 0.12680126167833805,
      "mlm": 1.934798389673233,
      "distill": 1.8773230910301208,
      "teacher_acc": 0.72244031727314
    },
    {
      "step": 1111,
      "lr": 4.974488426978454e-05,
      "elapsed_s": 831.7060656547546,
      "loss": 4.574786186218262,
      "hidden": 0.3081897348165512,
      "layer": 0.1299586445093155,
      "mlm": 1.9377944469451904,
      "distill": 1.9556329548358917,
      "teacher_acc": 0.7167164385318756
    },
    {
      "step": 1121,
      "lr": 4.973765998627628e-05,
      "elapsed_s": 839.099362373352,
      "loss": 4.719651937484741,
      "hidden": 0.3076632544398308,
      "layer": 0.12780077382922173,
      "mlm": 2.0092593729496,
      "distill": 2.031165659427643,
      "teacher_acc": 0.7114860415458679
    },
    {
      "step": 1131,
      "lr": 4.973033537780279e-05,
      "elapsed_s": 846.5289509296417,
      "loss": 4.613595247268677,
      "hidden": 0.307561457157135,
      "layer": 0.12706381268799305,
      "mlm": 1.9707413017749786,
      "distill": 1.964199185371399,
      "teacher_acc": 0.730304017663002
    },
    {
      "step": 1141,
      "lr": 4.972291047406945e-05,
      "elapsed_s": 853.9660637378693,
      "loss": 4.410772800445557,
      "hidden": 0.3032796233892441,
      "layer": 0.13063329830765724,
      "mlm": 1.8852495551109314,
      "distill": 1.8536473512649536,
      "teacher_acc": 0.7212417870759964
    },
    {
      "step": 1151,
      "lr": 4.971538530518836e-05,
      "elapsed_s": 861.391731262207,
      "loss": 4.5778995752334595,
      "hidden": 0.313470721244812,
      "layer": 0.132009856402874,
      "mlm": 1.9603157043457031,
      "distill": 1.9246373772621155,
      "teacher_acc": 0.7229406833648682
    },
    {
      "step": 1161,
      "lr": 4.970775990167826e-05,
      "elapsed_s": 868.8102362155914,
      "loss": 4.359367728233337,
      "hidden": 0.28991588950157166,
      "layer": 0.12484918721020222,
      "mlm": 1.8987728655338287,
      "distill": 1.8183383643627167,
      "teacher_acc": 0.7215947806835175
    },
    {
      "step": 1171,
      "lr": 4.970003429446441e-05,
      "elapsed_s": 876.1895446777344,
      "loss": 4.480438947677612,
      "hidden": 0.3037787601351738,
      "layer": 0.12765572220087051,
      "mlm": 1.90505912899971,
      "distill": 1.9039945900440216,
      "teacher_acc": 0.7179454863071442
    },
    {
      "step": 1181,
      "lr": 4.9692208514878444e-05,
      "elapsed_s": 883.5766868591309,
      "loss": 4.525385618209839,
      "hidden": 0.29506581276655197,
      "layer": 0.12710702419281006,
      "mlm": 1.882359355688095,
      "distill": 1.989341139793396,
      "teacher_acc": 0.7336955517530441
    },
    {
      "step": 1191,
      "lr": 4.9684282594658246e-05,
      "elapsed_s": 890.9544034004211,
      "loss": 4.677729249000549,
      "hidden": 0.3106038495898247,
      "layer": 0.13045357912778854,
      "mlm": 2.011395901441574,
      "distill": 1.9798988103866577,
      "teacher_acc": 0.7057065963745117
    },
    {
      "step": 1201,
      "lr": 4.967625656594782e-05,
      "elapsed_s": 898.3799946308136,
      "loss": 4.49394965171814,
      "hidden": 0.30099181830883026,
      "layer": 0.12817841582000256,
      "mlm": 1.9290640652179718,
      "distill": 1.8988126814365387,
      "teacher_acc": 0.7291355133056641
    },
    {
      "step": 1211,
      "lr": 4.9668130461297194e-05,
      "elapsed_s": 905.7777628898621,
      "loss": 4.50913679599762,
      "hidden": 0.3010120913386345,
      "layer": 0.12571420893073082,
      "mlm": 2.0044189393520355,
      "distill": 1.8398365676403046,
      "teacher_acc": 0.7036506235599518
    },
    {
      "step": 1221,
      "lr": 4.965990431366222e-05,
      "elapsed_s": 913.1580653190613,
      "loss": 4.343609809875488,
      "hidden": 0.2936936542391777,
      "layer": 0.12521342001855373,
      "mlm": 1.8477296829223633,
      "distill": 1.845886081457138,
      "teacher_acc": 0.7416622936725616
    },
    {
      "step": 1231,
      "lr": 4.965157815640452e-05,
      "elapsed_s": 920.537565946579,
      "loss": 4.40191376209259,
      "hidden": 0.30053282529115677,
      "layer": 0.1260622087866068,
      "mlm": 1.9028975665569305,
      "distill": 1.8349193632602692,
      "teacher_acc": 0.721520870923996
    },
    {
      "step": 1241,
      "lr": 4.964315202329127e-05,
      "elapsed_s": 927.9259502887726,
      "loss": 4.472258567810059,
      "hidden": 0.29379475861787796,
      "layer": 0.12663817405700684,
      "mlm": 1.9231075644493103,
      "distill": 1.8982424139976501,
      "teacher_acc": 0.7300374209880829
    },
    {
      "step": 1251,
      "lr": 4.963462594849515e-05,
      "elapsed_s": 935.2698395252228,
      "loss": 4.389858603477478,
      "hidden": 0.29641395807266235,
      "layer": 0.12565509416162968,
      "mlm": 1.909010797739029,
      "distill": 1.8251924514770508,
      "teacher_acc": 0.729375347495079
    },
    {
      "step": 1261,
      "lr": 4.962599996659411e-05,
      "elapsed_s": 942.6686508655548,
      "loss": 4.639959931373596,
      "hidden": 0.2991560325026512,
      "layer": 0.12722802348434925,
      "mlm": 1.9903516173362732,
      "distill": 1.9876822233200073,
      "teacher_acc": 0.7156456708908081
    },
    {
      "step": 1271,
      "lr": 4.961727411257131e-05,
      "elapsed_s": 950.0319755077362,
      "loss": 4.3744460344314575,
      "hidden": 0.3020690158009529,
      "layer": 0.12644479982554913,
      "mlm": 1.8198013305664062,
      "distill": 1.8872843384742737,
      "teacher_acc": 0.7496918588876724
    },
    {
      "step": 1281,
      "lr": 4.960844842181494e-05,
      "elapsed_s": 957.4452042579651,
      "loss": 4.513927221298218,
      "hidden": 0.29486898332834244,
      "layer": 0.1251782663166523,
      "mlm": 1.9579129815101624,
      "distill": 1.9036871790885925,
      "teacher_acc": 0.7145192176103592
    },
    {
      "step": 1291,
      "lr": 4.9599522930118096e-05,
      "elapsed_s": 964.8351647853851,
      "loss": 4.2646883726119995,
      "hidden": 0.2876865044236183,
      "layer": 0.12215254455804825,
      "mlm": 1.933357834815979,
      "distill": 1.69488126039505,
      "teacher_acc": 0.7047304958105087
    },
    {
      "step": 1301,
      "lr": 4.959049767367859e-05,
      "elapsed_s": 972.2409627437592,
      "loss": 4.248990058898926,
      "hidden": 0.28627555817365646,
      "layer": 0.12341926433146,
      "mlm": 1.848913162946701,
      "distill": 1.76581609249115,
      "teacher_acc": 0.7315411418676376
    },
    {
      "step": 1311,
      "lr": 4.958137268909887e-05,
      "elapsed_s": 979.6475322246552,
      "loss": 4.268232703208923,
      "hidden": 0.28770801424980164,
      "layer": 0.1230434700846672,
      "mlm": 1.856235146522522,
      "distill": 1.7750597894191742,
      "teacher_acc": 0.7229520380496979
    },
    {
      "step": 1321,
      "lr": 4.957214801338581e-05,
      "elapsed_s": 987.006005525589,
      "loss": 4.183823823928833,
      "hidden": 0.2891813889145851,
      "layer": 0.12435024045407772,
      "mlm": 1.7533661425113678,
      "distill": 1.789919674396515,
      "teacher_acc": 0.7429679185152054
    },
    {
      "step": 1331,
      "lr": 4.95628236839506e-05,
      "elapsed_s": 994.3906586170197,
      "loss": 4.391822457313538,
      "hidden": 0.3051561489701271,
      "layer": 0.12650306895375252,
      "mlm": 1.8430744409561157,
      "distill": 1.8751842081546783,
      "teacher_acc": 0.7304579168558121
    },
    {
      "step": 1341,
      "lr": 4.9553399738608586e-05,
      "elapsed_s": 1001.7643473148346,
      "loss": 4.4735177755355835,
      "hidden": 0.2943750470876694,
      "layer": 0.12432482466101646,
      "mlm": 1.9398589134216309,
      "distill": 1.882746309041977,
      "teacher_acc": 0.708781361579895
    },
    {
      "step": 1351,
      "lr": 4.954387621557911e-05,
      "elapsed_s": 1009.1344065666199,
      "loss": 4.336280822753906,
      "hidden": 0.3002393916249275,
      "layer": 0.12461583688855171,
      "mlm": 1.838879942893982,
      "distill": 1.8346141278743744,
      "teacher_acc": 0.7299322783946991
    },
    {
      "step": 1361,
      "lr": 4.953425315348534e-05,
      "elapsed_s": 1016.5312852859497,
      "loss": 4.320904970169067,
      "hidden": 0.2915872633457184,
      "layer": 0.12485742382705212,
      "mlm": 1.911872237920761,
      "distill": 1.7634294033050537,
      "teacher_acc": 0.7190799713134766
    },
    {
      "step": 1371,
      "lr": 4.9524530591354153e-05,
      "elapsed_s": 1023.9014797210693,
      "loss": 4.285800814628601,
      "hidden": 0.28473374992609024,
      "layer": 0.12136291898787022,
      "mlm": 1.9213156998157501,
      "distill": 1.7343361973762512,
      "teacher_acc": 0.715947300195694
    },
    {
      "step": 1381,
      "lr": 4.951470856861594e-05,
      "elapsed_s": 1031.2963905334473,
      "loss": 4.242612719535828,
      "hidden": 0.28850770741701126,
      "layer": 0.1227881945669651,
      "mlm": 1.8719923496246338,
      "distill": 1.7322107553482056,
      "teacher_acc": 0.7223887294530869
    },
    {
      "step": 1391,
      "lr": 4.950478712510446e-05,
      "elapsed_s": 1038.6915748119354,
      "loss": 4.047649502754211,
      "hidden": 0.2824060767889023,
      "layer": 0.11906011030077934,
      "mlm": 1.8015377819538116,
      "distill": 1.6217696070671082,
      "teacher_acc": 0.7295505702495575
    },
    {
      "step": 1401,
      "lr": 4.949476630105669e-05,
      "elapsed_s": 1046.0751898288727,
      "loss": 4.132641673088074,
      "hidden": 0.2914177179336548,
      "layer": 0.12169065326452255,
      "mlm": 1.8041305541992188,
      "distill": 1.684830367565155,
      "teacher_acc": 0.7275697588920593
    },
    {
      "step": 1411,
      "lr": 4.948464613711264e-05,
      "elapsed_s": 1053.4401726722717,
      "loss": 4.129916191101074,
      "hidden": 0.2892899289727211,
      "layer": 0.12262328527867794,
      "mlm": 1.758442461490631,
      "distill": 1.7315822541713715,
      "teacher_acc": 0.731286033987999
    },
    {
      "step": 1421,
      "lr": 4.947442667431522e-05,
      "elapsed_s": 1060.8332786560059,
      "loss": 4.327285408973694,
      "hidden": 0.2897583767771721,
      "layer": 0.1237189881503582,
      "mlm": 1.8594854176044464,
      "distill": 1.8264236450195312,
      "teacher_acc": 0.7237403392791748
    },
    {
      "step": 1431,
      "lr": 4.946410795410999e-05,
      "elapsed_s": 1068.1984114646912,
      "loss": 4.270741820335388,
      "hidden": 0.2866842597723007,
      "layer": 0.12094557844102383,
      "mlm": 1.8097352087497711,
      "distill": 1.8271653354167938,
      "teacher_acc": 0.7390832155942917
    },
    {
      "step": 1441,
      "lr": 4.9453690018345144e-05,
      "elapsed_s": 1075.5824282169342,
      "loss": 4.262914776802063,
      "hidden": 0.28310947865247726,
      "layer": 0.12107419595122337,
      "mlm": 1.8635119795799255,
      "distill": 1.7726466953754425,
      "teacher_acc": 0.7270894199609756
    },
    {
      "step": 1451,
      "lr": 4.944317290927117e-05,
      "elapsed_s": 1082.9756622314453,
      "loss": 4.196645975112915,
      "hidden": 0.28404614329338074,
      "layer": 0.12049952521920204,
      "mlm": 1.8071829974651337,
      "distill": 1.7611209452152252,
      "teacher_acc": 0.7384281158447266
    },
    {
      "step": 1461,
      "lr": 4.9432556669540816e-05,
      "elapsed_s": 1090.364845752716,
      "loss": 4.263148665428162,
      "hidden": 0.2885754257440567,
      "layer": 0.12266804836690426,
      "mlm": 1.8273625075817108,
      "distill": 1.797301173210144,
      "teacher_acc": 0.7298999726772308
    },
    {
      "step": 1471,
      "lr": 4.942184134220882e-05,
      "elapsed_s": 1097.748853445053,
      "loss": 4.262672662734985,
      "hidden": 0.2894189730286598,
      "layer": 0.12277720682322979,
      "mlm": 1.815561205148697,
      "distill": 1.8068848550319672,
      "teacher_acc": 0.7339396178722382
    },
    {
      "step": 1481,
      "lr": 4.9411026970731805e-05,
      "elapsed_s": 1105.1400890350342,
      "loss": 4.258927702903748,
      "hidden": 0.27983321994543076,
      "layer": 0.11972370743751526,
      "mlm": 1.8708434104919434,
      "distill": 1.7685560286045074,
      "teacher_acc": 0.716232419013977
    },
    {
      "step": 1491,
      "lr": 4.940011359896805e-05,
      "elapsed_s": 1112.508715391159,
      "loss": 4.23661482334137,
      "hidden": 0.28377529978752136,
      "layer": 0.12191220931708813,
      "mlm": 1.8968423306941986,
      "distill": 1.711265742778778,
      "teacher_acc": 0.7108533084392548
    },
    {
      "step": 1501,
      "lr": 4.9389101271177355e-05,
      "elapsed_s": 1119.8994617462158,
      "loss": 4.1674264669418335,
      "hidden": 0.2815132513642311,
      "layer": 0.11923379078507423,
      "mlm": 1.86190864443779,
      "distill": 1.682874321937561,
      "teacher_acc": 0.726785197854042
    },
    {
      "step": 1511,
      "lr": 4.937799003202081e-05,
      "elapsed_s": 1127.292000055313,
      "loss": 4.115518569946289,
      "hidden": 0.2762071117758751,
      "layer": 0.11728795245289803,
      "mlm": 1.8300577998161316,
      "distill": 1.6744024753570557,
      "teacher_acc": 0.7161758095026016
    },
    {
      "step": 1521,
      "lr": 4.93667799265607e-05,
      "elapsed_s": 1134.695062160492,
      "loss": 4.14449417591095,
      "hidden": 0.28291136026382446,
      "layer": 0.11974113434553146,
      "mlm": 1.814038246870041,
      "distill": 1.704762488603592,
      "teacher_acc": 0.7303779274225235
    },
    {
      "step": 1531,
      "lr": 4.9355471000260215e-05,
      "elapsed_s": 1142.0873663425446,
      "loss": 4.22127890586853,
      "hidden": 0.27795346081256866,
      "layer": 0.12069458328187466,
      "mlm": 1.805231511592865,
      "distill": 1.7997932434082031,
      "teacher_acc": 0.7420413345098495
    },
    {
      "step": 1541,
      "lr": 4.934406329898334e-05,
      "elapsed_s": 1149.478263616562,
      "loss": 4.197100579738617,
      "hidden": 0.27748703956604004,
      "layer": 0.11953026615083218,
      "mlm": 1.825773447751999,
      "distill": 1.7565878629684448,
      "teacher_acc": 0.7246889024972916
    },
    {
      "step": 1551,
      "lr": 4.933255686899465e-05,
      "elapsed_s": 1156.863220691681,
      "loss": 4.316618204116821,
      "hidden": 0.281481608748436,
      "layer": 0.1197182908654213,
      "mlm": 1.8745201230049133,
      "distill": 1.8192757070064545,
      "teacher_acc": 0.7336137890815735
    },
    {
      "step": 1561,
      "lr": 4.932095175695911e-05,
      "elapsed_s": 1164.292986869812,
      "loss": 4.202737927436829,
      "hidden": 0.27803388983011246,
      "layer": 0.11921351030468941,
      "mlm": 1.8015567660331726,
      "distill": 1.785506546497345,
      "teacher_acc": 0.7446371465921402
    },
    {
      "step": 1571,
      "lr": 4.9309248009941914e-05,
      "elapsed_s": 1171.6855747699738,
      "loss": 4.072049975395203,
      "hidden": 0.27580320835113525,
      "layer": 0.11885002069175243,
      "mlm": 1.8012520372867584,
      "distill": 1.659766435623169,
      "teacher_acc": 0.722566545009613
    },
    {
      "step": 1581,
      "lr": 4.929744567540826e-05,
      "elapsed_s": 1179.1023712158203,
      "loss": 4.242192983627319,
      "hidden": 0.2769583612680435,
      "layer": 0.11931121908128262,
      "mlm": 1.804571807384491,
      "distill": 1.8240488171577454,
      "teacher_acc": 0.7337799817323685
    },
    {
      "step": 1591,
      "lr": 4.9285544801223164e-05,
      "elapsed_s": 1186.5166730880737,
      "loss": 4.119462251663208,
      "hidden": 0.27202603220939636,
      "layer": 0.11788270249962807,
      "mlm": 1.7844215035438538,
      "distill": 1.732047289609909,
      "teacher_acc": 0.7386694997549057
    },
    {
      "step": 1601,
      "lr": 4.92735454356513e-05,
      "elapsed_s": 1193.9223265647888,
      "loss": 3.9569131731987,
      "hidden": 0.2660151943564415,
      "layer": 0.11577095463871956,
      "mlm": 1.795245736837387,
      "distill": 1.5717514753341675,
      "teacher_acc": 0.720261737704277
    },
    {
      "step": 1611,
      "lr": 4.9261447627356766e-05,
      "elapsed_s": 1201.325041770935,
      "loss": 4.203479766845703,
      "hidden": 0.2755109891295433,
      "layer": 0.11892695724964142,
      "mlm": 1.8778530359268188,
      "distill": 1.715141236782074,
      "teacher_acc": 0.7072772681713104
    },
    {
      "step": 1621,
      "lr": 4.92492514254029e-05,
      "elapsed_s": 1208.7398793697357,
      "loss": 4.1245410442352295,
      "hidden": 0.27370329946279526,
      "layer": 0.11781332828104496,
      "mlm": 1.7782695293426514,
      "distill": 1.739958256483078,
      "teacher_acc": 0.7412877231836319
    },
    {
      "step": 1631,
      "lr": 4.923695687925207e-05,
      "elapsed_s": 1216.1386933326721,
      "loss": 4.042859792709351,
      "hidden": 0.2672458216547966,
      "layer": 0.11648105457425117,
      "mlm": 1.8428515791893005,
      "distill": 1.6072761118412018,
      "teacher_acc": 0.7136448621749878
    },
    {
      "step": 1641,
      "lr": 4.922456403876552e-05,
      "elapsed_s": 1223.5526456832886,
      "loss": 4.069103121757507,
      "hidden": 0.27583178877830505,
      "layer": 0.11882580257952213,
      "mlm": 1.82866370677948,
      "distill": 1.6293628811836243,
      "teacher_acc": 0.709219291806221
    },
    {
      "step": 1651,
      "lr": 4.9212072954203086e-05,
      "elapsed_s": 1230.958993434906,
      "loss": 4.141493916511536,
      "hidden": 0.2743382155895233,
      "layer": 0.11745981499552727,
      "mlm": 1.8946268558502197,
      "distill": 1.6394605934619904,
      "teacher_acc": 0.6988830268383026
    },
    {
      "step": 1661,
      "lr": 4.919948367622307e-05,
      "elapsed_s": 1238.3738043308258,
      "loss": 3.971095621585846,
      "hidden": 0.27348922938108444,
      "layer": 0.1171498354524374,
      "mlm": 1.758977323770523,
      "distill": 1.606564998626709,
      "teacher_acc": 0.7259180545806885
    },
    {
      "step": 1671,
      "lr": 4.9186796255882e-05,
      "elapsed_s": 1245.7514443397522,
      "loss": 4.0444910526275635,
      "hidden": 0.27669356018304825,
      "layer": 0.11808189004659653,
      "mlm": 1.799822449684143,
      "distill": 1.6322405934333801,
      "teacher_acc": 0.7212954610586166
    },
    {
      "step": 1681,
      "lr": 4.917401074463441e-05,
      "elapsed_s": 1253.157922744751,
      "loss": 4.025470852851868,
      "hidden": 0.26970671117305756,
      "layer": 0.11674328707158566,
      "mlm": 1.7841910421848297,
      "distill": 1.6434946954250336,
      "teacher_acc": 0.7226119190454483
    },
    {
      "step": 1691,
      "lr": 4.9161127194332655e-05,
      "elapsed_s": 1260.5539331436157,
      "loss": 4.006395578384399,
      "hidden": 0.271147221326828,
      "layer": 0.1160933617502451,
      "mlm": 1.7752127647399902,
      "distill": 1.630841612815857,
      "teacher_acc": 0.726735070347786
    },
    {
      "step": 1701,
      "lr": 4.914814565722671e-05,
      "elapsed_s": 1267.9618742465973,
      "loss": 4.040911018848419,
      "hidden": 0.2694307118654251,
      "layer": 0.11729244142770767,
      "mlm": 1.7320196330547333,
      "distill": 1.7113837003707886,
      "teacher_acc": 0.7495951056480408
    },
    {
      "step": 1711,
      "lr": 4.9135066185963905e-05,
      "elapsed_s": 1275.3815183639526,
      "loss": 4.10949170589447,
      "hidden": 0.2712763324379921,
      "layer": 0.1159052699804306,
      "mlm": 1.8807397782802582,
      "distill": 1.628246545791626,
      "teacher_acc": 0.7094578593969345
    },
    {
      "step": 1721,
      "lr": 4.9121888833588795e-05,
      "elapsed_s": 1282.7949073314667,
      "loss": 3.8459686040878296,
      "hidden": 0.2645316757261753,
      "layer": 0.11463603936135769,
      "mlm": 1.7096973955631256,
      "distill": 1.5498898923397064,
      "teacher_acc": 0.7298763543367386
    },
    {
      "step": 1731,
      "lr": 4.9108613653542834e-05,
      "elapsed_s": 1290.212470293045,
      "loss": 4.060302257537842,
      "hidden": 0.2778415232896805,
      "layer": 0.11850542947649956,
      "mlm": 1.7755380868911743,
      "distill": 1.6698284447193146,
      "teacher_acc": 0.7308552116155624
    },
    {
      "step": 1741,
      "lr": 4.909524069966429e-05,
      "elapsed_s": 1297.5737907886505,
      "loss": 3.9033786058425903,
      "hidden": 0.27210283279418945,
      "layer": 0.11676839552819729,
      "mlm": 1.676222324371338,
      "distill": 1.6245664358139038,
      "teacher_acc": 0.7353321462869644
    },
    {
      "step": 1751,
      "lr": 4.9081770026187914e-05,
      "elapsed_s": 1304.9438817501068,
      "loss": 3.9643682837486267,
      "hidden": 0.268108993768692,
      "layer": 0.11562482826411724,
      "mlm": 1.7291994392871857,
      "distill": 1.6411384046077728,
      "teacher_acc": 0.742116704583168
    },
    {
      "step": 1761,
      "lr": 4.906820168774477e-05,
      "elapsed_s": 1312.3041315078735,
      "loss": 4.055130958557129,
      "hidden": 0.2741355299949646,
      "layer": 0.1160981860011816,
      "mlm": 1.7884034216403961,
      "distill": 1.660407394170761,
      "teacher_acc": 0.7300701141357422
    },
    {
      "step": 1771,
      "lr": 4.905453573936203e-05,
      "elapsed_s": 1319.7072315216064,
      "loss": 3.8725470900535583,
      "hidden": 0.25886010378599167,
      "layer": 0.11326215602457523,
      "mlm": 1.7237524092197418,
      "distill": 1.5744433999061584,
      "teacher_acc": 0.7289312928915024
    },
    {
      "step": 1781,
      "lr": 4.9040772236462695e-05,
      "elapsed_s": 1327.0891602039337,
      "loss": 3.910508930683136,
      "hidden": 0.26110412180423737,
      "layer": 0.11456430330872536,
      "mlm": 1.7168515026569366,
      "distill": 1.614167034626007,
      "teacher_acc": 0.7252945452928543
    },
    {
      "step": 1791,
      "lr": 4.902691123486544e-05,
      "elapsed_s": 1334.494841337204,
      "loss": 3.925660252571106,
      "hidden": 0.26289714127779007,
      "layer": 0.11490749008953571,
      "mlm": 1.7499323785305023,
      "distill": 1.5924798548221588,
      "teacher_acc": 0.7337967157363892
    },
    {
      "step": 1801,
      "lr": 4.901295279078431e-05,
      "elapsed_s": 1341.924957036972,
      "loss": 3.943657338619232,
      "hidden": 0.263511061668396,
      "layer": 0.11480100266635418,
      "mlm": 1.7710242867469788,
      "distill": 1.5882104337215424,
      "teacher_acc": 0.7232167720794678
    },
    {
      "step": 1811,
      "lr": 4.8998896960828576e-05,
      "elapsed_s": 1349.3606050014496,
      "loss": 4.007841229438782,
      "hidden": 0.26178329437971115,
      "layer": 0.11514490842819214,
      "mlm": 1.8540367782115936,
      "distill": 1.5726653635501862,
      "teacher_acc": 0.7053327709436417
    },
    {
      "step": 1821,
      "lr": 4.898474380200244e-05,
      "elapsed_s": 1356.791417837143,
      "loss": 4.013287663459778,
      "hidden": 0.2653432711958885,
      "layer": 0.11379984952509403,
      "mlm": 1.7653261721134186,
      "distill": 1.660375028848648,
      "teacher_acc": 0.7330894768238068
    },
    {
      "step": 1831,
      "lr": 4.8970493371704826e-05,
      "elapsed_s": 1364.2015180587769,
      "loss": 4.096006631851196,
      "hidden": 0.26983965188264847,
      "layer": 0.11775527149438858,
      "mlm": 1.8329637944698334,
      "distill": 1.6644857823848724,
      "teacher_acc": 0.7163500040769577
    },
    {
      "step": 1841,
      "lr": 4.8956145727729156e-05,
      "elapsed_s": 1371.6180520057678,
      "loss": 3.9370800256729126,
      "hidden": 0.26880139857530594,
      "layer": 0.1149628758430481,
      "mlm": 1.7297857999801636,
      "distill": 1.6122100353240967,
      "teacher_acc": 0.7322975099086761
    },
    {
      "step": 1851,
      "lr": 4.89417009282631e-05,
      "elapsed_s": 1379.0152342319489,
      "loss": 3.9829391837120056,
      "hidden": 0.26645859330892563,
      "layer": 0.11466134339570999,
      "mlm": 1.792383998632431,
      "distill": 1.600307434797287,
      "teacher_acc": 0.7116580754518509
    },
    {
      "step": 1861,
      "lr": 4.892715903188836e-05,
      "elapsed_s": 1386.3855543136597,
      "loss": 4.02520489692688,
      "hidden": 0.26583026349544525,
      "layer": 0.11542770266532898,
      "mlm": 1.84133842587471,
      "distill": 1.5944920480251312,
      "teacher_acc": 0.7134247869253159
    },
    {
      "step": 1871,
      "lr": 4.8912520097580395e-05,
      "elapsed_s": 1393.7514412403107,
      "loss": 4.0547550320625305,
      "hidden": 0.2741224393248558,
      "layer": 0.11564958281815052,
      "mlm": 1.8349095582962036,
      "distill": 1.6137757897377014,
      "teacher_acc": 0.7154436558485031
    },
    {
      "step": 1881,
      "lr": 4.889778418470823e-05,
      "elapsed_s": 1401.11745595932,
      "loss": 3.873760998249054,
      "hidden": 0.26062145084142685,
      "layer": 0.11372656188905239,
      "mlm": 1.7475957870483398,
      "distill": 1.5480589866638184,
      "teacher_acc": 0.7312120199203491
    },
    {
      "step": 1891,
      "lr": 4.888295135303418e-05,
      "elapsed_s": 1408.4978585243225,
      "loss": 3.8677785992622375,
      "hidden": 0.2633001543581486,
      "layer": 0.11530493013560772,
      "mlm": 1.7338138818740845,
      "distill": 1.54971182346344,
      "teacher_acc": 0.7224676758050919
    },
    {
      "step": 1901,
      "lr": 4.886802166271364e-05,
      "elapsed_s": 1415.885833978653,
      "loss": 3.8258262872695923,
      "hidden": 0.2571691647171974,
      "layer": 0.11422812938690186,
      "mlm": 1.6562677919864655,
      "distill": 1.5981061160564423,
      "teacher_acc": 0.7442539632320404
    },
    {
      "step": 1911,
      "lr": 4.88529951742948e-05,
      "elapsed_s": 1423.2395277023315,
      "loss": 4.137242794036865,
      "hidden": 0.2623438760638237,
      "layer": 0.11479996144771576,
      "mlm": 1.9038341641426086,
      "distill": 1.6513210535049438,
      "teacher_acc": 0.7084044516086578
    },
    {
      "step": 1921,
      "lr": 4.883787194871841e-05,
      "elapsed_s": 1430.6223375797272,
      "loss": 3.931863307952881,
      "hidden": 0.267056405544281,
      "layer": 0.11772950552403927,
      "mlm": 1.7514340579509735,
      "distill": 1.5874518156051636,
      "teacher_acc": 0.7247260808944702
    },
    {
      "step": 1931,
      "lr": 4.8822652047317576e-05,
      "elapsed_s": 1437.9897689819336,
      "loss": 3.9127307534217834,
      "hidden": 0.26528214663267136,
      "layer": 0.1145928930491209,
      "mlm": 1.6910908818244934,
      "distill": 1.6337791979312897,
      "teacher_acc": 0.7463971078395844
    },
    {
      "step": 1941,
      "lr": 4.880733553181745e-05,
      "elapsed_s": 1445.4078583717346,
      "loss": 3.8677561283111572,
      "hidden": 0.26494765281677246,
      "layer": 0.11483337730169296,
      "mlm": 1.674789547920227,
      "distill": 1.605654537677765,
      "teacher_acc": 0.7425475418567657
    },
    {
      "step": 1951,
      "lr": 4.8791922464335015e-05,
      "elapsed_s": 1452.7917742729187,
      "loss": 3.912749230861664,
      "hidden": 0.26438158005476,
      "layer": 0.11647258140146732,
      "mlm": 1.7126152217388153,
      "distill": 1.6131345331668854,
      "teacher_acc": 0.7387824803590775
    },
    {
      "step": 1961,
      "lr": 4.877641290737884e-05,
      "elapsed_s": 1460.1799867153168,
      "loss": 4.000911474227905,
      "hidden": 0.2635009437799454,
      "layer": 0.11493142880499363,
      "mlm": 1.8374084532260895,
      "distill": 1.5790354013442993,
      "teacher_acc": 0.7120477259159088
    },
    {
      "step": 1971,
      "lr": 4.87608069238488e-05,
      "elapsed_s": 1467.5589821338654,
      "loss": 3.952027440071106,
      "hidden": 0.25954311341047287,
      "layer": 0.11294668540358543,
      "mlm": 1.7966808080673218,
      "distill": 1.5797869861125946,
      "teacher_acc": 0.7198255211114883
    },
    {
      "step": 1981,
      "lr": 4.874510457703583e-05,
      "elapsed_s": 1474.9765911102295,
      "loss": 3.734243333339691,
      "hidden": 0.2588770315051079,
      "layer": 0.11292687244713306,
      "mlm": 1.6978994607925415,
      "distill": 1.4621263444423676,
      "teacher_acc": 0.7278008013963699
    },
    {
      "step": 1991,
      "lr": 4.872930593062168e-05,
      "elapsed_s": 1482.3742113113403,
      "loss": 3.7141858339309692,
      "hidden": 0.25460510328412056,
      "layer": 0.11156057566404343,
      "mlm": 1.655147910118103,
      "distill": 1.4940473437309265,
      "teacher_acc": 0.7379762977361679
    },
    {
      "step": 2001,
      "lr": 4.8713411048678635e-05,
      "elapsed_s": 1491.76766872406,
      "loss": 3.9283209443092346,
      "hidden": 0.25907303765416145,
      "layer": 0.11318337358534336,
      "mlm": 1.7712864577770233,
      "distill": 1.5822968184947968,
      "teacher_acc": 0.7172820121049881
    },
    {
      "step": 2011,
      "lr": 4.8697419995669304e-05,
      "elapsed_s": 1499.1479773521423,
      "loss": 3.985919237136841,
      "hidden": 0.259055458009243,
      "layer": 0.11354961059987545,
      "mlm": 1.833306461572647,
      "distill": 1.577727049589157,
      "teacher_acc": 0.712208017706871
    },
    {
      "step": 2021,
      "lr": 4.868133283644627e-05,
      "elapsed_s": 1506.5302166938782,
      "loss": 3.741446614265442,
      "hidden": 0.24916762486100197,
      "layer": 0.11203144676983356,
      "mlm": 1.7160248458385468,
      "distill": 1.471070796251297,
      "teacher_acc": 0.7306434512138367
    },
    {
      "step": 2031,
      "lr": 4.866514963625192e-05,
      "elapsed_s": 1513.9140689373016,
      "loss": 3.912960648536682,
      "hidden": 0.2593256011605263,
      "layer": 0.11219154857099056,
      "mlm": 1.725127249956131,
      "distill": 1.6130864322185516,
      "teacher_acc": 0.7276073694229126
    },
    {
      "step": 2041,
      "lr": 4.864887046071813e-05,
      "elapsed_s": 1521.3447892665863,
      "loss": 3.858840227127075,
      "hidden": 0.25858693569898605,
      "layer": 0.11235799640417099,
      "mlm": 1.6772578060626984,
      "distill": 1.608229637145996,
      "teacher_acc": 0.7404515445232391
    },
    {
      "step": 2051,
      "lr": 4.8632495375866004e-05,
      "elapsed_s": 1528.7397742271423,
      "loss": 3.7260199189186096,
      "hidden": 0.25173715874552727,
      "layer": 0.11114678718149662,
      "mlm": 1.7014999389648438,
      "distill": 1.465472251176834,
      "teacher_acc": 0.7315275371074677
    },
    {
      "step": 2061,
      "lr": 4.8616024448105626e-05,
      "elapsed_s": 1536.2033755779266,
      "loss": 3.7934157848358154,
      "hidden": 0.25040078535676,
      "layer": 0.11035937070846558,
      "mlm": 1.7161026298999786,
      "distill": 1.5213318765163422,
      "teacher_acc": 0.7229634076356888
    },
    {
      "step": 2071,
      "lr": 4.859945774423575e-05,
      "elapsed_s": 1543.624699831009,
      "loss": 3.8551888465881348,
      "hidden": 0.25906796008348465,
      "layer": 0.11276037245988846,
      "mlm": 1.7273021638393402,
      "distill": 1.5533705651760101,
      "teacher_acc": 0.7239642888307571
    },
    {
      "step": 2081,
      "lr": 4.858279533144358e-05,
      "elapsed_s": 1551.0422327518463,
      "loss": 3.7497435212135315,
      "hidden": 0.24814024195075035,
      "layer": 0.11147261038422585,
      "mlm": 1.7492544949054718,
      "distill": 1.448472261428833,
      "teacher_acc": 0.725109875202179
    },
    {
      "step": 2091,
      "lr": 4.856603727730447e-05,
      "elapsed_s": 1558.4539713859558,
      "loss": 3.7135396003723145,
      "hidden": 0.2477710247039795,
      "layer": 0.1095570120960474,
      "mlm": 1.7603477835655212,
      "distill": 1.4028713703155518,
      "teacher_acc": 0.7150020599365234
    },
    {
      "step": 2101,
      "lr": 4.8549183649781626e-05,
      "elapsed_s": 1565.8681509494781,
      "loss": 3.778154730796814,
      "hidden": 0.2488444447517395,
      "layer": 0.10881204903125763,
      "mlm": 1.7663404941558838,
      "distill": 1.4597193002700806,
      "teacher_acc": 0.7076955437660217
    },
    {
      "step": 2111,
      "lr": 4.853223451722589e-05,
      "elapsed_s": 1573.2999246120453,
      "loss": 3.961124897003174,
      "hidden": 0.2597087174654007,
      "layer": 0.11299195699393749,
      "mlm": 1.7284411191940308,
      "distill": 1.6567704677581787,
      "teacher_acc": 0.7349591553211212
    },
    {
      "step": 2121,
      "lr": 4.8515189948375434e-05,
      "elapsed_s": 1580.6742188930511,
      "loss": 3.8618377447128296,
      "hidden": 0.2569420374929905,
      "layer": 0.11181840486824512,
      "mlm": 1.748778611421585,
      "distill": 1.543265849351883,
      "teacher_acc": 0.72219517827034
    },
    {
      "step": 2131,
      "lr": 4.849805001235544e-05,
      "elapsed_s": 1588.077444076538,
      "loss": 3.893868327140808,
      "hidden": 0.25733304768800735,
      "layer": 0.11332016624510288,
      "mlm": 1.7597509026527405,
      "distill": 1.5627912878990173,
      "teacher_acc": 0.7182294279336929
    },
    {
      "step": 2141,
      "lr": 4.8480814778677886e-05,
      "elapsed_s": 1595.4836041927338,
      "loss": 3.752512037754059,
      "hidden": 0.2519802264869213,
      "layer": 0.1101859100162983,
      "mlm": 1.7296915650367737,
      "distill": 1.4637670814990997,
      "teacher_acc": 0.7298867702484131
    },
    {
      "step": 2151,
      "lr": 4.8463484317241226e-05,
      "elapsed_s": 1602.8707621097565,
      "loss": 3.6804771423339844,
      "hidden": 0.24470311030745506,
      "layer": 0.10954228788614273,
      "mlm": 1.6736562848091125,
      "distill": 1.4626435935497284,
      "teacher_acc": 0.7283311635255814
    },
    {
      "step": 2161,
      "lr": 4.8446058698330115e-05,
      "elapsed_s": 1610.2833743095398,
      "loss": 3.830751061439514,
      "hidden": 0.26048295199871063,
      "layer": 0.11235157772898674,
      "mlm": 1.753524899482727,
      "distill": 1.5000845193862915,
      "teacher_acc": 0.7219198048114777
    },
    {
      "step": 2171,
      "lr": 4.842853799261513e-05,
      "elapsed_s": 1617.6941299438477,
      "loss": 3.819101631641388,
      "hidden": 0.2561846449971199,
      "layer": 0.11272404342889786,
      "mlm": 1.7824145555496216,
      "distill": 1.4679557085037231,
      "teacher_acc": 0.7075820714235306
    },
    {
      "step": 2181,
      "lr": 4.841092227115247e-05,
      "elapsed_s": 1625.0718519687653,
      "loss": 3.888596475124359,
      "hidden": 0.2566503472626209,
      "layer": 0.11228472739458084,
      "mlm": 1.777033507823944,
      "distill": 1.5421198308467865,
      "teacher_acc": 0.7178526520729065
    },
    {
      "step": 2191,
      "lr": 4.839321160538369e-05,
      "elapsed_s": 1632.4810466766357,
      "loss": 3.810259997844696,
      "hidden": 0.25594790279865265,
      "layer": 0.11116081103682518,
      "mlm": 1.751077264547348,
      "distill": 1.4917064607143402,
      "teacher_acc": 0.7177864909172058
    },
    {
      "step": 2201,
      "lr": 4.837540606713538e-05,
      "elapsed_s": 1639.8658628463745,
      "loss": 3.7928138971328735,
      "hidden": 0.24622415006160736,
      "layer": 0.11025381833314896,
      "mlm": 1.7675414681434631,
      "distill": 1.4776972532272339,
      "teacher_acc": 0.7079342603683472
    },
    {
      "step": 2211,
      "lr": 4.8357505728618886e-05,
      "elapsed_s": 1647.2524738311768,
      "loss": 3.7466330528259277,
      "hidden": 0.2507931999862194,
      "layer": 0.10913983918726444,
      "mlm": 1.779582291841507,
      "distill": 1.4108944237232208,
      "teacher_acc": 0.7024128586053848
    },
    {
      "step": 2221,
      "lr": 4.8339510662430046e-05,
      "elapsed_s": 1654.6791286468506,
      "loss": 3.839081585407257,
      "hidden": 0.24698897451162338,
      "layer": 0.1100331787019968,
      "mlm": 1.8070143163204193,
      "distill": 1.4830726385116577,
      "teacher_acc": 0.7061618268489838
    },
    {
      "step": 2231,
      "lr": 4.832142094154886e-05,
      "elapsed_s": 1662.0784311294556,
      "loss": 3.811948299407959,
      "hidden": 0.25376270711421967,
      "layer": 0.1114901565015316,
      "mlm": 1.6992760002613068,
      "distill": 1.549401730298996,
      "teacher_acc": 0.7325648665428162
    },
    {
      "step": 2241,
      "lr": 4.83032366393392e-05,
      "elapsed_s": 1669.5000557899475,
      "loss": 3.9008437991142273,
      "hidden": 0.2512689679861069,
      "layer": 0.11265360005199909,
      "mlm": 1.7716874778270721,
      "distill": 1.5702916383743286,
      "teacher_acc": 0.7235399931669235
    },
    {
      "step": 2251,
      "lr": 4.828495782954852e-05,
      "elapsed_s": 1676.9206519126892,
      "loss": 3.7354241609573364,
      "hidden": 0.2521323710680008,
      "layer": 0.11011753603816032,
      "mlm": 1.736986368894577,
      "distill": 1.439114362001419,
      "teacher_acc": 0.7176721841096878
    },
    {
      "step": 2261,
      "lr": 4.826658458630756e-05,
      "elapsed_s": 1684.3310117721558,
      "loss": 3.8716242909431458,
      "hidden": 0.258310966193676,
      "layer": 0.11187855340540409,
      "mlm": 1.7341187000274658,
      "distill": 1.564944326877594,
      "teacher_acc": 0.7295988500118256
    },
    {
      "step": 2271,
      "lr": 4.8248116984130034e-05,
      "elapsed_s": 1691.7226912975311,
      "loss": 3.731353759765625,
      "hidden": 0.25206349045038223,
      "layer": 0.11220180429518223,
      "mlm": 1.7203965187072754,
      "distill": 1.450729250907898,
      "teacher_acc": 0.7182573825120926
    },
    {
      "step": 2281,
      "lr": 4.822955509791233e-05,
      "elapsed_s": 1699.1242909431458,
      "loss": 3.761576235294342,
      "hidden": 0.24819886311888695,
      "layer": 0.1087129395455122,
      "mlm": 1.7110427618026733,
      "distill": 1.4997792541980743,
      "teacher_acc": 0.7260957807302475
    },
    {
      "step": 2291,
      "lr": 4.8210899002933235e-05,
      "elapsed_s": 1706.518870830536,
      "loss": 3.745406925678253,
      "hidden": 0.24332544952630997,
      "layer": 0.10934064909815788,
      "mlm": 1.7383477687835693,
      "distill": 1.4657378494739532,
      "teacher_acc": 0.7136690765619278
    },
    {
      "step": 2301,
      "lr": 4.819214877485358e-05,
      "elapsed_s": 1713.938626050949,
      "loss": 3.960775673389435,
      "hidden": 0.2648190073668957,
      "layer": 0.11514735035598278,
      "mlm": 1.7090501487255096,
      "distill": 1.6645138263702393,
      "teacher_acc": 0.737593024969101
    },
    {
      "step": 2311,
      "lr": 4.8173304489715953e-05,
      "elapsed_s": 1721.3406410217285,
      "loss": 3.642211079597473,
      "hidden": 0.2459014095366001,
      "layer": 0.10843637771904469,
      "mlm": 1.6925817131996155,
      "distill": 1.4036083221435547,
      "teacher_acc": 0.7297173589468002
    },
    {
      "step": 2321,
      "lr": 4.815436622394441e-05,
      "elapsed_s": 1728.7602188587189,
      "loss": 3.7932201623916626,
      "hidden": 0.24933533370494843,
      "layer": 0.10909699648618698,
      "mlm": 1.7553210258483887,
      "distill": 1.4846799075603485,
      "teacher_acc": 0.7145818024873734
    },
    {
      "step": 2331,
      "lr": 4.813533405434417e-05,
      "elapsed_s": 1736.188850402832,
      "loss": 3.704191207885742,
      "hidden": 0.2512279227375984,
      "layer": 0.11084950156509876,
      "mlm": 1.7046216428279877,
      "distill": 1.4416889548301697,
      "teacher_acc": 0.7253220230340958
    },
    {
      "step": 2341,
      "lr": 4.8116208058101236e-05,
      "elapsed_s": 1743.610481262207,
      "loss": 3.863282024860382,
      "hidden": 0.24838250502943993,
      "layer": 0.10942491888999939,
      "mlm": 1.7685328125953674,
      "distill": 1.5432717204093933,
      "teacher_acc": 0.7260880172252655
    },
    {
      "step": 2351,
      "lr": 4.8096988312782174e-05,
      "elapsed_s": 1751.0236730575562,
      "loss": 3.6840537786483765,
      "hidden": 0.24761249870061874,
      "layer": 0.10832477547228336,
      "mlm": 1.7780731916427612,
      "distill": 1.3565932512283325,
      "teacher_acc": 0.6999672204256058
    },
    {
      "step": 2361,
      "lr": 4.8077674896333725e-05,
      "elapsed_s": 1758.4118888378143,
      "loss": 3.6843072175979614,
      "hidden": 0.24846041202545166,
      "layer": 0.1089798305183649,
      "mlm": 1.6926963925361633,
      "distill": 1.440200001001358,
      "teacher_acc": 0.7290272563695908
    },
    {
      "step": 2371,
      "lr": 4.805826788708254e-05,
      "elapsed_s": 1765.8164603710175,
      "loss": 3.603092133998871,
      "hidden": 0.24257640168070793,
      "layer": 0.10641664266586304,
      "mlm": 1.6400000751018524,
      "distill": 1.4247309565544128,
      "teacher_acc": 0.7381000220775604
    },
    {
      "step": 2381,
      "lr": 4.803876736373483e-05,
      "elapsed_s": 1773.2482841014862,
      "loss": 3.6311395168304443,
      "hidden": 0.24065001308918,
      "layer": 0.10753550939261913,
      "mlm": 1.726937621831894,
      "distill": 1.369134098291397,
      "teacher_acc": 0.7084744423627853
    },
    {
      "step": 2391,
      "lr": 4.801917340537605e-05,
      "elapsed_s": 1780.655324459076,
      "loss": 3.840295970439911,
      "hidden": 0.24767836928367615,
      "layer": 0.10932904295623302,
      "mlm": 1.8044228553771973,
      "distill": 1.4858517944812775,
      "teacher_acc": 0.703942209482193
    },
    {
      "step": 2401,
      "lr": 4.799948609147061e-05,
      "elapsed_s": 1788.0623397827148,
      "loss": 3.6096293926239014,
      "hidden": 0.24044471234083176,
      "layer": 0.10646991059184074,
      "mlm": 1.6577262580394745,
      "distill": 1.4177787005901337,
      "teacher_acc": 0.7344533950090408
    },
    {
      "step": 2411,
      "lr": 4.79797055018615e-05,
      "elapsed_s": 1795.4843757152557,
      "loss": 3.6840533018112183,
      "hidden": 0.24279526248574257,
      "layer": 0.10737080127000809,
      "mlm": 1.6824278831481934,
      "distill": 1.462349385023117,
      "teacher_acc": 0.7308273017406464
    },
    {
      "step": 2421,
      "lr": 4.795983171677002e-05,
      "elapsed_s": 1802.9127252101898,
      "loss": 3.804903745651245,
      "hidden": 0.24725133180618286,
      "layer": 0.10980324260890484,
      "mlm": 1.7106261253356934,
      "distill": 1.54487344622612,
      "teacher_acc": 0.7355051040649414
    },
    {
      "step": 2431,
      "lr": 4.7939864816795416e-05,
      "elapsed_s": 1810.3432569503784,
      "loss": 3.7888460755348206,
      "hidden": 0.24837645888328552,
      "layer": 0.10931314714252949,
      "mlm": 1.7285901010036469,
      "distill": 1.5088465213775635,
      "teacher_acc": 0.7219253033399582
    },
    {
      "step": 2441,
      "lr": 4.791980488291456e-05,
      "elapsed_s": 1817.7673411369324,
      "loss": 3.7419418692588806,
      "hidden": 0.24069231376051903,
      "layer": 0.10797939263284206,
      "mlm": 1.7442275285720825,
      "distill": 1.462339997291565,
      "teacher_acc": 0.7189342528581619
    },
    {
      "step": 2451,
      "lr": 4.7899651996481656e-05,
      "elapsed_s": 1825.1733677387238,
      "loss": 3.761119782924652,
      "hidden": 0.25064223632216454,
      "layer": 0.1095963567495346,
      "mlm": 1.699464350938797,
      "distill": 1.5055726766586304,
      "teacher_acc": 0.7272935211658478
    },
    {
      "step": 2461,
      "lr": 4.787940623922784e-05,
      "elapsed_s": 1832.5688915252686,
      "loss": 3.6708080172538757,
      "hidden": 0.24238311126828194,
      "layer": 0.10794962011277676,
      "mlm": 1.7068127393722534,
      "distill": 1.4252543151378632,
      "teacher_acc": 0.7142873704433441
    },
    {
      "step": 2471,
      "lr": 4.7859067693260926e-05,
      "elapsed_s": 1839.965213060379,
      "loss": 3.5825703144073486,
      "hidden": 0.2462666556239128,
      "layer": 0.10680276341736317,
      "mlm": 1.592526137828827,
      "distill": 1.4441094100475311,
      "teacher_acc": 0.7498723566532135
    },
    {
      "step": 2481,
      "lr": 4.783863644106502e-05,
      "elapsed_s": 1847.3670659065247,
      "loss": 3.59833163022995,
      "hidden": 0.2421642579138279,
      "layer": 0.10684924758970737,
      "mlm": 1.6500813364982605,
      "distill": 1.4104971587657928,
      "teacher_acc": 0.7275962233543396
    },
    {
      "step": 2491,
      "lr": 4.781811256550023e-05,
      "elapsed_s": 1854.78395485878,
      "loss": 3.68978214263916,
      "hidden": 0.24138816446065903,
      "layer": 0.10864538140594959,
      "mlm": 1.6910502314567566,
      "distill": 1.4616328477859497,
      "teacher_acc": 0.7209601551294327
    },
    {
      "step": 2501,
      "lr": 4.7797496149802256e-05,
      "elapsed_s": 1862.185598373413,
      "loss": 3.7686917781829834,
      "hidden": 0.24645867198705673,
      "layer": 0.10835725255310535,
      "mlm": 1.7570590674877167,
      "distill": 1.4645367860794067,
      "teacher_acc": 0.7163214534521103
    },
    {
      "step": 2511,
      "lr": 4.7776787277582134e-05,
      "elapsed_s": 1869.598785161972,
      "loss": 3.7397080659866333,
      "hidden": 0.24499952048063278,
      "layer": 0.10806623473763466,
      "mlm": 1.7814353108406067,
      "distill": 1.4142406284809113,
      "teacher_acc": 0.7064929753541946
    },
    {
      "step": 2521,
      "lr": 4.7755986032825864e-05,
      "elapsed_s": 1876.9911653995514,
      "loss": 3.681542158126831,
      "hidden": 0.2533358745276928,
      "layer": 0.10940774530172348,
      "mlm": 1.6334728002548218,
      "distill": 1.4866937398910522,
      "teacher_acc": 0.7419825494289398
    },
    {
      "step": 2531,
      "lr": 4.773509249989406e-05,
      "elapsed_s": 1884.4329330921173,
      "loss": 3.831621766090393,
      "hidden": 0.24358056858181953,
      "layer": 0.10879357345402241,
      "mlm": 1.7595036327838898,
      "distill": 1.5305601954460144,
      "teacher_acc": 0.7233162820339203
    },
    {
      "step": 2541,
      "lr": 4.771410676352159e-05,
      "elapsed_s": 1891.8787107467651,
      "loss": 3.644809365272522,
      "hidden": 0.24500878155231476,
      "layer": 0.10855513252317905,
      "mlm": 1.5861805379390717,
      "distill": 1.514333724975586,
      "teacher_acc": 0.7467977553606033
    },
    {
      "step": 2551,
      "lr": 4.7693028908817315e-05,
      "elapsed_s": 1899.2731022834778,
      "loss": 3.6844981908798218,
      "hidden": 0.24672172591090202,
      "layer": 0.10882776416838169,
      "mlm": 1.7102844417095184,
      "distill": 1.4263564050197601,
      "teacher_acc": 0.7162268310785294
    },
    {
      "step": 2561,
      "lr": 4.767185902126364e-05,
      "elapsed_s": 1906.6529474258423,
      "loss": 3.5618661642074585,
      "hidden": 0.2363271489739418,
      "layer": 0.10503928922116756,
      "mlm": 1.7053262293338776,
      "distill": 1.3313659131526947,
      "teacher_acc": 0.7192791402339935
    },
    {
      "step": 2571,
      "lr": 4.765059718671623e-05,
      "elapsed_s": 1914.0563731193542,
      "loss": 3.596912682056427,
      "hidden": 0.24243199452757835,
      "layer": 0.10740852542221546,
      "mlm": 1.6463701725006104,
      "distill": 1.411974310874939,
      "teacher_acc": 0.7432228177785873
    },
    {
      "step": 2581,
      "lr": 4.7629243491403636e-05,
      "elapsed_s": 1921.4137330055237,
      "loss": 3.560342848300934,
      "hidden": 0.23930377885699272,
      "layer": 0.10598771274089813,
      "mlm": 1.674494743347168,
      "distill": 1.354246735572815,
      "teacher_acc": 0.7177818864583969
    },
    {
      "step": 2591,
      "lr": 4.760779802192698e-05,
      "elapsed_s": 1928.7610535621643,
      "loss": 3.5935680866241455,
      "hidden": 0.23820922896265984,
      "layer": 0.10677331313490868,
      "mlm": 1.6324653327465057,
      "distill": 1.4312976002693176,
      "teacher_acc": 0.7370318174362183
    },
    {
      "step": 2601,
      "lr": 4.758626086525956e-05,
      "elapsed_s": 1936.1127738952637,
      "loss": 3.7478145956993103,
      "hidden": 0.24265775457024574,
      "layer": 0.10825501941144466,
      "mlm": 1.800310879945755,
      "distill": 1.4080606698989868,
      "teacher_acc": 0.7111178785562515
    },
    {
      "step": 2611,
      "lr": 4.756463210874652e-05,
      "elapsed_s": 1943.4745259284973,
      "loss": 3.5550922751426697,
      "hidden": 0.2391643114387989,
      "layer": 0.1057338397949934,
      "mlm": 1.5180002450942993,
      "distill": 1.505896508693695,
      "teacher_acc": 0.7615246325731277
    },
    {
      "step": 2621,
      "lr": 4.754291184010449e-05,
      "elapsed_s": 1950.8625221252441,
      "loss": 3.58420729637146,
      "hidden": 0.23577246069908142,
      "layer": 0.10539300180971622,
      "mlm": 1.6794508397579193,
      "distill": 1.3805151283740997,
      "teacher_acc": 0.7280884981155396
    },
    {
      "step": 2631,
      "lr": 4.752110014742124e-05,
      "elapsed_s": 1958.2600302696228,
      "loss": 3.723642945289612,
      "hidden": 0.23914989084005356,
      "layer": 0.10667040199041367,
      "mlm": 1.7678002715110779,
      "distill": 1.4242076575756073,
      "teacher_acc": 0.7110923379659653
    },
    {
      "step": 2641,
      "lr": 4.74991971191553e-05,
      "elapsed_s": 1965.6535589694977,
      "loss": 3.7126502990722656,
      "hidden": 0.24177956208586693,
      "layer": 0.10700585879385471,
      "mlm": 1.693537563085556,
      "distill": 1.4820507764816284,
      "teacher_acc": 0.7239854037761688
    },
    {
      "step": 2651,
      "lr": 4.7477202844135646e-05,
      "elapsed_s": 1973.0942614078522,
      "loss": 3.5955039262771606,
      "hidden": 0.24167544767260551,
      "layer": 0.10688715986907482,
      "mlm": 1.6716265976428986,
      "distill": 1.3870827853679657,
      "teacher_acc": 0.7232551872730255
    },
    {
      "step": 2661,
      "lr": 4.745511741156129e-05,
      "elapsed_s": 1980.5191600322723,
      "loss": 3.6259926557540894,
      "hidden": 0.23596793040633202,
      "layer": 0.10512718930840492,
      "mlm": 1.69126096367836,
      "distill": 1.410232275724411,
      "teacher_acc": 0.722245380282402
    },
    {
      "step": 2671,
      "lr": 4.7432940911000923e-05,
      "elapsed_s": 1987.9092147350311,
      "loss": 3.6282278299331665,
      "hidden": 0.2408795990049839,
      "layer": 0.10780544392764568,
      "mlm": 1.6640565395355225,
      "distill": 1.4285092949867249,
      "teacher_acc": 0.7319728881120682
    },
    {
      "step": 2681,
      "lr": 4.7410673432392596e-05,
      "elapsed_s": 1995.3141012191772,
      "loss": 3.5768274664878845,
      "hidden": 0.23598787188529968,
      "layer": 0.10724631138145924,
      "mlm": 1.678175687789917,
      "distill": 1.3730529248714447,
      "teacher_acc": 0.7206576019525528
    },
    {
      "step": 2691,
      "lr": 4.738831506604332e-05,
      "elapsed_s": 2002.7380542755127,
      "loss": 3.7140308022499084,
      "hidden": 0.242545984685421,
      "layer": 0.10712094977498055,
      "mlm": 1.7198494672775269,
      "distill": 1.4555288255214691,
      "teacher_acc": 0.7228926718235016
    },
    {
      "step": 2701,
      "lr": 4.7365865902628684e-05,
      "elapsed_s": 2010.1523413658142,
      "loss": 3.4395722150802612,
      "hidden": 0.23167967796325684,
      "layer": 0.10434584505856037,
      "mlm": 1.6193323731422424,
      "distill": 1.304707556962967,
      "teacher_acc": 0.7343793958425522
    },
    {
      "step": 2711,
      "lr": 4.734332603319255e-05,
      "elapsed_s": 2017.5735819339752,
      "loss": 3.6597050428390503,
      "hidden": 0.23803844302892685,
      "layer": 0.10731230489909649,
      "mlm": 1.6911435723304749,
      "distill": 1.4388284981250763,
      "teacher_acc": 0.7284220308065414
    },
    {
      "step": 2721,
      "lr": 4.732069554914661e-05,
      "elapsed_s": 2024.9937336444855,
      "loss": 3.6472243070602417,
      "hidden": 0.2409161627292633,
      "layer": 0.10627330653369427,
      "mlm": 1.7463038265705109,
      "distill": 1.3659515082836151,
      "teacher_acc": 0.7127766609191895
    },
    {
      "step": 2731,
      "lr": 4.7297974542270055e-05,
      "elapsed_s": 2032.4183926582336,
      "loss": 3.631434142589569,
      "hidden": 0.238052599132061,
      "layer": 0.10654278472065926,
      "mlm": 1.633485198020935,
      "distill": 1.4685722887516022,
      "teacher_acc": 0.7346589714288712
    },
    {
      "step": 2741,
      "lr": 4.72751631047092e-05,
      "elapsed_s": 2039.7783660888672,
      "loss": 3.6866583228111267,
      "hidden": 0.23772570863366127,
      "layer": 0.10672623664140701,
      "mlm": 1.7171207964420319,
      "distill": 1.4407230019569397,
      "teacher_acc": 0.7275938540697098
    },
    {
      "step": 2751,
      "lr": 4.7252261328977115e-05,
      "elapsed_s": 2047.1711785793304,
      "loss": 3.6036709547042847,
      "hidden": 0.2395215667784214,
      "layer": 0.10622072778642178,
      "mlm": 1.6842427849769592,
      "distill": 1.3872746229171753,
      "teacher_acc": 0.7253856509923935
    },
    {
      "step": 2761,
      "lr": 4.7229269307953235e-05,
      "elapsed_s": 2054.5901930332184,
      "loss": 3.532498836517334,
      "hidden": 0.23626259714365005,
      "layer": 0.10546763986349106,
      "mlm": 1.6397127211093903,
      "distill": 1.3675271570682526,
      "teacher_acc": 0.733916237950325
    },
    {
      "step": 2771,
      "lr": 4.7206187134882977e-05,
      "elapsed_s": 2061.9925315380096,
      "loss": 3.5738848447799683,
      "hidden": 0.2349291294813156,
      "layer": 0.10499691031873226,
      "mlm": 1.6095466613769531,
      "distill": 1.4419815242290497,
      "teacher_acc": 0.7304109036922455
    },
    {
      "step": 2781,
      "lr": 4.71830149033774e-05,
      "elapsed_s": 2069.407297849655,
      "loss": 3.580619513988495,
      "hidden": 0.23565727844834328,
      "layer": 0.10646986775100231,
      "mlm": 1.6680371761322021,
      "distill": 1.3880327939987183,
      "teacher_acc": 0.72722327709198
    },
    {
      "step": 2791,
      "lr": 4.715975270741278e-05,
      "elapsed_s": 2076.8249473571777,
      "loss": 3.5825352668762207,
      "hidden": 0.23516757041215897,
      "layer": 0.10478775762021542,
      "mlm": 1.6737564504146576,
      "distill": 1.3860498070716858,
      "teacher_acc": 0.7210666686296463
    },
    {
      "step": 2801,
      "lr": 4.713640064133025e-05,
      "elapsed_s": 2084.247176170349,
      "loss": 3.664033591747284,
      "hidden": 0.23736470565199852,
      "layer": 0.10661087185144424,
      "mlm": 1.7583079040050507,
      "distill": 1.3776907324790955,
      "teacher_acc": 0.7080092877149582
    },
    {
      "step": 2811,
      "lr": 4.7112958799835424e-05,
      "elapsed_s": 2091.6675131320953,
      "loss": 3.617379665374756,
      "hidden": 0.2411416582763195,
      "layer": 0.10608030296862125,
      "mlm": 1.659661889076233,
      "distill": 1.4223942756652832,
      "teacher_acc": 0.7266990840435028
    },
    {
      "step": 2821,
      "lr": 4.708942727799801e-05,
      "elapsed_s": 2099.090729236603,
      "loss": 3.5019719004631042,
      "hidden": 0.23186643049120903,
      "layer": 0.1036705020815134,
      "mlm": 1.6292818784713745,
      "distill": 1.3571219742298126,
      "teacher_acc": 0.7291790544986725
    },
    {
      "step": 2831,
      "lr": 4.7065806171251416e-05,
      "elapsed_s": 2106.4805195331573,
      "loss": 3.7564882040023804,
      "hidden": 0.24046286568045616,
      "layer": 0.10817813500761986,
      "mlm": 1.7259016036987305,
      "distill": 1.4955718219280243,
      "teacher_acc": 0.7274407595396042
    },
    {
      "step": 2841,
      "lr": 4.7042095575392355e-05,
      "elapsed_s": 2113.897171497345,
      "loss": 3.5006067156791687,
      "hidden": 0.23034659028053284,
      "layer": 0.10453379340469837,
      "mlm": 1.6120376884937286,
      "distill": 1.3756089806556702,
      "teacher_acc": 0.7391497492790222
    },
    {
      "step": 2851,
      "lr": 4.701829558658046e-05,
      "elapsed_s": 2121.3115117549896,
      "loss": 3.4552191495895386,
      "hidden": 0.2289695106446743,
      "layer": 0.10405125841498375,
      "mlm": 1.6536909639835358,
      "distill": 1.291563630104065,
      "teacher_acc": 0.7207521945238113
    },
    {
      "step": 2861,
      "lr": 4.699440630133794e-05,
      "elapsed_s": 2128.734341621399,
      "loss": 3.503120541572571,
      "hidden": 0.23052410408854485,
      "layer": 0.10387588478624821,
      "mlm": 1.6073122024536133,
      "distill": 1.3828222453594208,
      "teacher_acc": 0.7410666197538376
    },
    {
      "step": 2871,
      "lr": 4.697042781654913e-05,
      "elapsed_s": 2136.164852619171,
      "loss": 3.4902254343032837,
      "hidden": 0.23738359659910202,
      "layer": 0.10568131506443024,
      "mlm": 1.5749092996120453,
      "distill": 1.3877082765102386,
      "teacher_acc": 0.7454861998558044
    },
    {
      "step": 2881,
      "lr": 4.694636022946012e-05,
      "elapsed_s": 2143.6112546920776,
      "loss": 3.5363218784332275,
      "hidden": 0.23062248155474663,
      "layer": 0.10387315787374973,
      "mlm": 1.6959924399852753,
      "distill": 1.3271478712558746,
      "teacher_acc": 0.7219624519348145
    },
    {
      "step": 2891,
      "lr": 4.692220363767833e-05,
      "elapsed_s": 2151.0236394405365,
      "loss": 3.6036964058876038,
      "hidden": 0.22718659043312073,
      "layer": 0.10374492965638638,
      "mlm": 1.6717808544635773,
      "distill": 1.4256699681282043,
      "teacher_acc": 0.7323247045278549
    },
    {
      "step": 2901,
      "lr": 4.68979581391722e-05,
      "elapsed_s": 2158.429487466812,
      "loss": 3.61070054769516,
      "hidden": 0.23945430293679237,
      "layer": 0.10574058815836906,
      "mlm": 1.7240363359451294,
      "distill": 1.354885309934616,
      "teacher_acc": 0.7143816351890564
    },
    {
      "step": 2911,
      "lr": 4.687362383227069e-05,
      "elapsed_s": 2165.8093507289886,
      "loss": 3.6366803646087646,
      "hidden": 0.23762637376785278,
      "layer": 0.10596693865954876,
      "mlm": 1.7215954959392548,
      "distill": 1.3868486881256104,
      "teacher_acc": 0.7206857204437256
    },
    {
      "step": 2921,
      "lr": 4.684920081566295e-05,
      "elapsed_s": 2173.2271959781647,
      "loss": 3.4630767703056335,
      "hidden": 0.2299012504518032,
      "layer": 0.10233664326369762,
      "mlm": 1.6417950689792633,
      "distill": 1.3103108406066895,
      "teacher_acc": 0.7232145220041275
    },
    {
      "step": 2931,
      "lr": 4.68246891883979e-05,
      "elapsed_s": 2180.6168296337128,
      "loss": 3.502602696418762,
      "hidden": 0.23003851622343063,
      "layer": 0.10482638515532017,
      "mlm": 1.6567935347557068,
      "distill": 1.3333189189434052,
      "teacher_acc": 0.7252501547336578
    },
    {
      "step": 2941,
      "lr": 4.680008904988379e-05,
      "elapsed_s": 2188.0033230781555,
      "loss": 3.491953432559967,
      "hidden": 0.23427829891443253,
      "layer": 0.10359874926507473,
      "mlm": 1.606177419424057,
      "distill": 1.365420013666153,
      "teacher_acc": 0.7423140704631805
    },
    {
      "step": 2951,
      "lr": 4.677540049988789e-05,
      "elapsed_s": 2195.373770236969,
      "loss": 3.580929458141327,
      "hidden": 0.23450344428420067,
      "layer": 0.10556199587881565,
      "mlm": 1.7018214762210846,
      "distill": 1.357320100069046,
      "teacher_acc": 0.7183585315942764
    },
    {
      "step": 2961,
      "lr": 4.6750623638535986e-05,
      "elapsed_s": 2202.7666988372803,
      "loss": 3.5984174013137817,
      "hidden": 0.2290143482387066,
      "layer": 0.10321427136659622,
      "mlm": 1.6942948997020721,
      "distill": 1.394486665725708,
      "teacher_acc": 0.7126488238573074
    },
    {
      "step": 2971,
      "lr": 4.6725758566312016e-05,
      "elapsed_s": 2210.151052713394,
      "loss": 3.4650648832321167,
      "hidden": 0.2291531153023243,
      "layer": 0.10461174137890339,
      "mlm": 1.6531810462474823,
      "distill": 1.3012717366218567,
      "teacher_acc": 0.7204356491565704
    },
    {
      "step": 2981,
      "lr": 4.6700805384057687e-05,
      "elapsed_s": 2217.5151166915894,
      "loss": 3.5106738209724426,
      "hidden": 0.22873886302113533,
      "layer": 0.10297016426920891,
      "mlm": 1.6726548373699188,
      "distill": 1.3290561437606812,
      "teacher_acc": 0.721130445599556
    },
    {
      "step": 2991,
      "lr": 4.667576419297202e-05,
      "elapsed_s": 2224.8559131622314,
      "loss": 3.5212265253067017,
      "hidden": 0.22997724637389183,
      "layer": 0.10374366492033005,
      "mlm": 1.734131783246994,
      "distill": 1.275268316268921,
      "teacher_acc": 0.702075332403183
    },
    {
      "step": 3001,
      "lr": 4.665063509461097e-05,
      "elapsed_s": 2234.205440759659,
      "loss": 3.4606210589408875,
      "hidden": 0.22931714355945587,
      "layer": 0.10343053191900253,
      "mlm": 1.6665265560150146,
      "distill": 1.2837449610233307,
      "teacher_acc": 0.7193577736616135
    },
    {
      "step": 3011,
      "lr": 4.6625418190887e-05,
      "elapsed_s": 2241.529851436615,
      "loss": 3.3743128180503845,
      "hidden": 0.2232120931148529,
      "layer": 0.10135133564472198,
      "mlm": 1.6456921696662903,
      "distill": 1.2315207719802856,
      "teacher_acc": 0.7201991677284241
    },
    {
      "step": 3021,
      "lr": 4.660011358406867e-05,
      "elapsed_s": 2248.904124736786,
      "loss": 3.508089780807495,
      "hidden": 0.2337881326675415,
      "layer": 0.10397909954190254,
      "mlm": 1.649567037820816,
      "distill": 1.3389569222927094,
      "teacher_acc": 0.7356233596801758
    },
    {
      "step": 3031,
      "lr": 4.657472137678024e-05,
      "elapsed_s": 2256.288508415222,
      "loss": 3.4837343096733093,
      "hidden": 0.22679223120212555,
      "layer": 0.10177144408226013,
      "mlm": 1.6783565282821655,
      "distill": 1.3009076118469238,
      "teacher_acc": 0.7154030948877335
    },
    {
      "step": 3041,
      "lr": 4.654924167200123e-05,
      "elapsed_s": 2263.6509714126587,
      "loss": 3.561523973941803,
      "hidden": 0.22662726417183876,
      "layer": 0.103306595236063,
      "mlm": 1.6692866086959839,
      "distill": 1.387329488992691,
      "teacher_acc": 0.7342425137758255
    },
    {
      "step": 3051,
      "lr": 4.6523674573066e-05,
      "elapsed_s": 2271.039669275284,
      "loss": 3.491649568080902,
      "hidden": 0.22500937804579735,
      "layer": 0.1024861540645361,
      "mlm": 1.6900636851787567,
      "distill": 1.3003240823745728,
      "teacher_acc": 0.7152795344591141
    },
    {
      "step": 3061,
      "lr": 4.649802018366336e-05,
      "elapsed_s": 2278.450541496277,
      "loss": 3.50148206949234,
      "hidden": 0.22964788973331451,
      "layer": 0.10274216905236244,
      "mlm": 1.6394510567188263,
      "distill": 1.3513641059398651,
      "teacher_acc": 0.7282019257545471
    },
    {
      "step": 3071,
      "lr": 4.647227860783612e-05,
      "elapsed_s": 2285.81498336792,
      "loss": 3.4740242958068848,
      "hidden": 0.2279980257153511,
      "layer": 0.10369544290006161,
      "mlm": 1.6022800505161285,
      "distill": 1.3639004528522491,
      "teacher_acc": 0.7424096465110779
    },
    {
      "step": 3081,
      "lr": 4.6446449949980665e-05,
      "elapsed_s": 2293.2138237953186,
      "loss": 3.6519100666046143,
      "hidden": 0.23669841513037682,
      "layer": 0.10585329681634903,
      "mlm": 1.7092592120170593,
      "distill": 1.416327327489853,
      "teacher_acc": 0.7229047417640686
    },
    {
      "step": 3091,
      "lr": 4.6420534314846585e-05,
      "elapsed_s": 2300.6238343715668,
      "loss": 3.3918741941452026,
      "hidden": 0.2239185944199562,
      "layer": 0.10104312188923359,
      "mlm": 1.6155100166797638,
      "distill": 1.2780054807662964,
      "teacher_acc": 0.723976656794548
    },
    {
      "step": 3101,
      "lr": 4.639453180753619e-05,
      "elapsed_s": 2308.028860092163,
      "loss": 3.4800446033477783,
      "hidden": 0.22441308945417404,
      "layer": 0.10252377390861511,
      "mlm": 1.665266752243042,
      "distill": 1.314689815044403,
      "teacher_acc": 0.7118273973464966
    },
    {
      "step": 3111,
      "lr": 4.63684425335041e-05,
      "elapsed_s": 2315.443920135498,
      "loss": 3.657502770423889,
      "hidden": 0.2322998382151127,
      "layer": 0.10514740087091923,
      "mlm": 1.7614023685455322,
      "distill": 1.3789271116256714,
      "teacher_acc": 0.7034724950790405
    },
    {
      "step": 3121,
      "lr": 4.6342266598556814e-05,
      "elapsed_s": 2322.8459804058075,
      "loss": 3.4586631059646606,
      "hidden": 0.23120060935616493,
      "layer": 0.10307871177792549,
      "mlm": 1.5992388129234314,
      "distill": 1.3454837501049042,
      "teacher_acc": 0.7405604720115662
    },
    {
      "step": 3131,
      "lr": 4.6316004108852305e-05,
      "elapsed_s": 2330.2439291477203,
      "loss": 3.499767541885376,
      "hidden": 0.2233915477991104,
      "layer": 0.10289429314434528,
      "mlm": 1.7268340587615967,
      "distill": 1.2747032344341278,
      "teacher_acc": 0.7004177868366241
    },
    {
      "step": 3141,
      "lr": 4.628965517089959e-05,
      "elapsed_s": 2337.658492088318,
      "loss": 3.5173186659812927,
      "hidden": 0.222753643989563,
      "layer": 0.10281359776854515,
      "mlm": 1.699946105480194,
      "distill": 1.3204584121704102,
      "teacher_acc": 0.7086157202720642
    },
    {
      "step": 3151,
      "lr": 4.6263219891558244e-05,
      "elapsed_s": 2345.087569952011,
      "loss": 3.600011706352234,
      "hidden": 0.22582751885056496,
      "layer": 0.10341009870171547,
      "mlm": 1.6581669747829437,
      "distill": 1.4384846985340118,
      "teacher_acc": 0.729967474937439
    },
    {
      "step": 3161,
      "lr": 4.6236698378038026e-05,
      "elapsed_s": 2352.4644508361816,
      "loss": 3.5091535449028015,
      "hidden": 0.23309017717838287,
      "layer": 0.10450707748532295,
      "mlm": 1.71028670668602,
      "distill": 1.2804329693317413,
      "teacher_acc": 0.7077615112066269
    },
    {
      "step": 3171,
      "lr": 4.621009073789842e-05,
      "elapsed_s": 2359.8673663139343,
      "loss": 3.527905583381653,
      "hidden": 0.22566555440425873,
      "layer": 0.10294582508504391,
      "mlm": 1.7181035280227661,
      "distill": 1.3069979846477509,
      "teacher_acc": 0.7122737467288971
    },
    {
      "step": 3181,
      "lr": 4.6183397079048205e-05,
      "elapsed_s": 2367.3015348911285,
      "loss": 3.487317204475403,
      "hidden": 0.23124703019857407,
      "layer": 0.10366122610867023,
      "mlm": 1.6682642698287964,
      "distill": 1.3047282695770264,
      "teacher_acc": 0.7175710648298264
    },
    {
      "step": 3191,
      "lr": 4.6156617509745015e-05,
      "elapsed_s": 2374.7088572978973,
      "loss": 3.4809818863868713,
      "hidden": 0.2208900861442089,
      "layer": 0.10162181593477726,
      "mlm": 1.6904520690441132,
      "distill": 1.2979387044906616,
      "teacher_acc": 0.7137667089700699
    },
    {
      "step": 3201,
      "lr": 4.6129752138594874e-05,
      "elapsed_s": 2382.118670463562,
      "loss": 3.4504796862602234,
      "hidden": 0.22447019815444946,
      "layer": 0.10228989645838737,
      "mlm": 1.6588618755340576,
      "distill": 1.2915324866771698,
      "teacher_acc": 0.7179579138755798
    },
    {
      "step": 3211,
      "lr": 4.6102801074551806e-05,
      "elapsed_s": 2389.5258524417877,
      "loss": 3.5374115705490112,
      "hidden": 0.2326372191309929,
      "layer": 0.10468166880309582,
      "mlm": 1.6603041291236877,
      "distill": 1.3594921231269836,
      "teacher_acc": 0.730151355266571
    },
    {
      "step": 3221,
      "lr": 4.607576442691737e-05,
      "elapsed_s": 2396.93758225441,
      "loss": 3.565000891685486,
      "hidden": 0.2248787060379982,
      "layer": 0.10271011479198933,
      "mlm": 1.697321355342865,
      "distill": 1.366567075252533,
      "teacher_acc": 0.721306636929512
    },
    {
      "step": 3231,
      "lr": 4.6048642305340185e-05,
      "elapsed_s": 2404.342048406601,
      "loss": 3.4903936982154846,
      "hidden": 0.22763410210609436,
      "layer": 0.10249729081988335,
      "mlm": 1.6350296139717102,
      "distill": 1.3488471806049347,
      "teacher_acc": 0.7368521392345428
    },
    {
      "step": 3241,
      "lr": 4.6021434819815555e-05,
      "elapsed_s": 2411.7597846984863,
      "loss": 3.5416045784950256,
      "hidden": 0.2282167673110962,
      "layer": 0.1037945169955492,
      "mlm": 1.6292797923088074,
      "distill": 1.4039939641952515,
      "teacher_acc": 0.7344960719347
    },
    {
      "step": 3251,
      "lr": 4.599414208068495e-05,
      "elapsed_s": 2419.182558774948,
      "loss": 3.419649362564087,
      "hidden": 0.23007922992110252,
      "layer": 0.10282059572637081,
      "mlm": 1.5672361552715302,
      "distill": 1.3408444821834564,
      "teacher_acc": 0.7406398355960846
    },
    {
      "step": 3261,
      "lr": 4.5966764198635606e-05,
      "elapsed_s": 2426.5875005722046,
      "loss": 3.4718623757362366,
      "hidden": 0.22298796847462654,
      "layer": 0.10260258428752422,
      "mlm": 1.672924816608429,
      "distill": 1.3016604483127594,
      "teacher_acc": 0.7142860144376755
    },
    {
      "step": 3271,
      "lr": 4.593930128470005e-05,
      "elapsed_s": 2433.978696346283,
      "loss": 3.5107290148735046,
      "hidden": 0.2212652713060379,
      "layer": 0.10115457512438297,
      "mlm": 1.715896487236023,
      "distill": 1.3017246723175049,
      "teacher_acc": 0.7148478478193283
    },
    {
      "step": 3281,
      "lr": 4.5911753450255665e-05,
      "elapsed_s": 2441.3789868354797,
      "loss": 3.58910071849823,
      "hidden": 0.23301169648766518,
      "layer": 0.10499644093215466,
      "mlm": 1.6986188292503357,
      "distill": 1.3719602227210999,
      "teacher_acc": 0.7117029875516891
    },
    {
      "step": 3291,
      "lr": 4.588412080702426e-05,
      "elapsed_s": 2448.7913999557495,
      "loss": 3.372100532054901,
      "hidden": 0.22256171703338623,
      "layer": 0.10064934007823467,
      "mlm": 1.607201725244522,
      "distill": 1.269450694322586,
      "teacher_acc": 0.7256952226161957
    },
    {
      "step": 3301,
      "lr": 4.5856403467071536e-05,
      "elapsed_s": 2456.2188000679016,
      "loss": 3.4336293935775757,
      "hidden": 0.22901742160320282,
      "layer": 0.10366160608828068,
      "mlm": 1.6359600722789764,
      "distill": 1.2878036499023438,
      "teacher_acc": 0.7136553823947906
    },
    {
      "step": 3311,
      "lr": 4.582860154280674e-05,
      "elapsed_s": 2463.6272122859955,
      "loss": 3.544716000556946,
      "hidden": 0.22044740244746208,
      "layer": 0.10249673761427402,
      "mlm": 1.694400042295456,
      "distill": 1.35817289352417,
      "teacher_acc": 0.7168470919132233
    },
    {
      "step": 3321,
      "lr": 4.580071514698211e-05,
      "elapsed_s": 2471.0372614860535,
      "loss": 3.3085479736328125,
      "hidden": 0.22407560795545578,
      "layer": 0.10275277495384216,
      "mlm": 1.5773303806781769,
      "distill": 1.231689989566803,
      "teacher_acc": 0.7351345121860504
    },
    {
      "step": 3331,
      "lr": 4.57727443926925e-05,
      "elapsed_s": 2478.4736206531525,
      "loss": 3.402520477771759,
      "hidden": 0.22688784077763557,
      "layer": 0.10335866920650005,
      "mlm": 1.5565273761749268,
      "distill": 1.340538114309311,
      "teacher_acc": 0.7515149861574173
    },
    {
      "step": 3341,
      "lr": 4.574468939337488e-05,
      "elapsed_s": 2485.8861334323883,
      "loss": 3.4884793162345886,
      "hidden": 0.2202131226658821,
      "layer": 0.10254723206162453,
      "mlm": 1.7354383170604706,
      "distill": 1.2613410651683807,
      "teacher_acc": 0.7056065946817398
    },
    {
      "step": 3351,
      "lr": 4.5716550262807854e-05,
      "elapsed_s": 2493.276671886444,
      "loss": 3.4397005438804626,
      "hidden": 0.22360840439796448,
      "layer": 0.10160521231591702,
      "mlm": 1.643522322177887,
      "distill": 1.2981588542461395,
      "teacher_acc": 0.7268181294202805
    },
    {
      "step": 3361,
      "lr": 4.568832711511125e-05,
      "elapsed_s": 2500.6512598991394,
      "loss": 3.588262915611267,
      "hidden": 0.2267148718237877,
      "layer": 0.10242516919970512,
      "mlm": 1.7419219017028809,
      "distill": 1.3416987359523773,
      "teacher_acc": 0.7026091068983078
    },
    {
      "step": 3371,
      "lr": 4.566002006474562e-05,
      "elapsed_s": 2508.05770945549,
      "loss": 3.514046549797058,
      "hidden": 0.23101866617798805,
      "layer": 0.10360232181847095,
      "mlm": 1.5809781551361084,
      "distill": 1.4192299246788025,
      "teacher_acc": 0.7460936456918716
    },
    {
      "step": 3381,
      "lr": 4.563162922651179e-05,
      "elapsed_s": 2515.4639484882355,
      "loss": 3.515875220298767,
      "hidden": 0.22301653772592545,
      "layer": 0.10282004997134209,
      "mlm": 1.6879366040229797,
      "distill": 1.3304955065250397,
      "teacher_acc": 0.7279070913791656
    },
    {
      "step": 3391,
      "lr": 4.5603154715550386e-05,
      "elapsed_s": 2522.895223379135,
      "loss": 3.4018912315368652,
      "hidden": 0.2239724136888981,
      "layer": 0.10192890465259552,
      "mlm": 1.6196550130844116,
      "distill": 1.2833269238471985,
      "teacher_acc": 0.7178000658750534
    },
    {
      "step": 3401,
      "lr": 4.557459664734141e-05,
      "elapsed_s": 2530.280441761017,
      "loss": 3.522800087928772,
      "hidden": 0.226976927369833,
      "layer": 0.10235772281885147,
      "mlm": 1.6863990128040314,
      "distill": 1.3312683999538422,
      "teacher_acc": 0.7239378839731216
    },
    {
      "step": 3411,
      "lr": 4.554595513770369e-05,
      "elapsed_s": 2537.67915892601,
      "loss": 3.5089078545570374,
      "hidden": 0.222249835729599,
      "layer": 0.10050370916724205,
      "mlm": 1.7329918444156647,
      "distill": 1.2811645567417145,
      "teacher_acc": 0.7053619176149368
    },
    {
      "step": 3421,
      "lr": 4.551723030279446e-05,
      "elapsed_s": 2545.1264295578003,
      "loss": 3.35947984457016,
      "hidden": 0.2204466313123703,
      "layer": 0.10075901821255684,
      "mlm": 1.581898272037506,
      "distill": 1.2863087356090546,
      "teacher_acc": 0.7301050275564194
    },
    {
      "step": 3431,
      "lr": 4.5488422259108907e-05,
      "elapsed_s": 2552.5427136421204,
      "loss": 3.3932032585144043,
      "hidden": 0.221500676125288,
      "layer": 0.10246274806559086,
      "mlm": 1.6904788613319397,
      "distill": 1.2084916830062866,
      "teacher_acc": 0.7053885459899902
    },
    {
      "step": 3441,
      "lr": 4.5459531123479676e-05,
      "elapsed_s": 2559.929175376892,
      "loss": 3.4944602847099304,
      "hidden": 0.23181834071874619,
      "layer": 0.10496977902948856,
      "mlm": 1.6038219034671783,
      "distill": 1.3745166659355164,
      "teacher_acc": 0.7321884483098984
    },
    {
      "step": 3451,
      "lr": 4.543055701307637e-05,
      "elapsed_s": 2567.320256948471,
      "loss": 3.3996405005455017,
      "hidden": 0.21922963112592697,
      "layer": 0.09974186681210995,
      "mlm": 1.6947680413722992,
      "distill": 1.2165422439575195,
      "teacher_acc": 0.7056132704019547
    },
    {
      "step": 3461,
      "lr": 4.540150004540512e-05,
      "elapsed_s": 2574.717602491379,
      "loss": 3.432230234146118,
      "hidden": 0.22933698445558548,
      "layer": 0.10330581292510033,
      "mlm": 1.5856387317180634,
      "distill": 1.3362646400928497,
      "teacher_acc": 0.7384810149669647
    },
    {
      "step": 3471,
      "lr": 4.537236033830809e-05,
      "elapsed_s": 2582.1194155216217,
      "loss": 3.3968421816825867,
      "hidden": 0.2132473811507225,
      "layer": 0.10007931850850582,
      "mlm": 1.6304021775722504,
      "distill": 1.2899055182933807,
      "teacher_acc": 0.7273385524749756
    },
    {
      "step": 3481,
      "lr": 4.534313800996299e-05,
      "elapsed_s": 2589.542094707489,
      "loss": 3.440521001815796,
      "hidden": 0.22472675889730453,
      "layer": 0.10329141095280647,
      "mlm": 1.676109790802002,
      "distill": 1.2633121013641357,
      "teacher_acc": 0.7177644520998001
    },
    {
      "step": 3491,
      "lr": 4.531383317888261e-05,
      "elapsed_s": 2596.9556863307953,
      "loss": 3.3939183950424194,
      "hidden": 0.21635373309254646,
      "layer": 0.10069276764988899,
      "mlm": 1.615008682012558,
      "distill": 1.2958558201789856,
      "teacher_acc": 0.7311837375164032
    },
    {
      "step": 3501,
      "lr": 4.528444596391433e-05,
      "elapsed_s": 2604.3900628089905,
      "loss": 3.3732004165649414,
      "hidden": 0.22191773727536201,
      "layer": 0.10196337662637234,
      "mlm": 1.5925023257732391,
      "distill": 1.2858809530735016,
      "teacher_acc": 0.7350538969039917
    },
    {
      "step": 3511,
      "lr": 4.525497648423964e-05,
      "elapsed_s": 2611.8201043605804,
      "loss": 3.463964104652405,
      "hidden": 0.21726492419838905,
      "layer": 0.10132593661546707,
      "mlm": 1.7119967937469482,
      "distill": 1.2667745351791382,
      "teacher_acc": 0.7097632735967636
    },
    {
      "step": 3521,
      "lr": 4.522542485937369e-05,
      "elapsed_s": 2619.2207708358765,
      "loss": 3.4572603702545166,
      "hidden": 0.2189236804842949,
      "layer": 0.10150349326431751,
      "mlm": 1.7127948105335236,
      "distill": 1.2558664083480835,
      "teacher_acc": 0.7084037959575653
    },
    {
      "step": 3531,
      "lr": 4.519579120916472e-05,
      "elapsed_s": 2626.6420307159424,
      "loss": 3.395686447620392,
      "hidden": 0.2208048291504383,
      "layer": 0.10194237530231476,
      "mlm": 1.632282555103302,
      "distill": 1.2708231210708618,
      "teacher_acc": 0.7240957915782928
    },
    {
      "step": 3541,
      "lr": 4.5166075653793676e-05,
      "elapsed_s": 2634.0398409366608,
      "loss": 3.405287504196167,
      "hidden": 0.21933967620134354,
      "layer": 0.10075551457703114,
      "mlm": 1.6624489724636078,
      "distill": 1.2537815570831299,
      "teacher_acc": 0.7112442851066589
    },
    {
      "step": 3551,
      "lr": 4.513627831377365e-05,
      "elapsed_s": 2641.4800169467926,
      "loss": 3.2774580121040344,
      "hidden": 0.21580269187688828,
      "layer": 0.10005523264408112,
      "mlm": 1.5768791139125824,
      "distill": 1.218945860862732,
      "teacher_acc": 0.7273680418729782
    },
    {
      "step": 3561,
      "lr": 4.510639930994942e-05,
      "elapsed_s": 2648.9230070114136,
      "loss": 3.535762667655945,
      "hidden": 0.22324201837182045,
      "layer": 0.10037110559642315,
      "mlm": 1.769502580165863,
      "distill": 1.2695905268192291,
      "teacher_acc": 0.7000212669372559
    },
    {
      "step": 3571,
      "lr": 4.507643876349698e-05,
      "elapsed_s": 2656.34974527359,
      "loss": 3.3849878907203674,
      "hidden": 0.22146416455507278,
      "layer": 0.1014827024191618,
      "mlm": 1.613898128271103,
      "distill": 1.2774201035499573,
      "teacher_acc": 0.7219107449054718
    },
    {
      "step": 3581,
      "lr": 4.5046396795922964e-05,
      "elapsed_s": 2663.7874853610992,
      "loss": 3.332967221736908,
      "hidden": 0.21942906081676483,
      "layer": 0.10010307841002941,
      "mlm": 1.6011481583118439,
      "distill": 1.242909461259842,
      "teacher_acc": 0.72371406853199
    },
    {
      "step": 3591,
      "lr": 4.501627352906429e-05,
      "elapsed_s": 2671.1968195438385,
      "loss": 3.43490993976593,
      "hidden": 0.2172873094677925,
      "layer": 0.09979121945798397,
      "mlm": 1.6759430766105652,
      "distill": 1.2744966447353363,
      "teacher_acc": 0.7079648226499557
    },
    {
      "step": 3601,
      "lr": 4.498606908508754e-05,
      "elapsed_s": 2678.596888780594,
      "loss": 3.4864338040351868,
      "hidden": 0.216410331428051,
      "layer": 0.10081839561462402,
      "mlm": 1.6856513917446136,
      "distill": 1.3175525069236755,
      "teacher_acc": 0.7179049700498581
    },
    {
      "step": 3611,
      "lr": 4.4955783586488514e-05,
      "elapsed_s": 2686.0373544692993,
      "loss": 3.3339715003967285,
      "hidden": 0.21926870197057724,
      "layer": 0.09914068505167961,
      "mlm": 1.645705908536911,
      "distill": 1.2001579105854034,
      "teacher_acc": 0.7183408141136169
    },
    {
      "step": 3621,
      "lr": 4.492541715609177e-05,
      "elapsed_s": 2693.4673421382904,
      "loss": 3.543579339981079,
      "hidden": 0.222660094499588,
      "layer": 0.10127940960228443,
      "mlm": 1.6892383396625519,
      "distill": 1.3583810925483704,
      "teacher_acc": 0.7298067510128021
    },
    {
      "step": 3631,
      "lr": 4.489496991705006e-05,
      "elapsed_s": 2700.8846101760864,
      "loss": 3.5348130464553833,
      "hidden": 0.2192484475672245,
      "layer": 0.10178992711007595,
      "mlm": 1.7294292449951172,
      "distill": 1.315991997718811,
      "teacher_acc": 0.7085979282855988
    },
    {
      "step": 3641,
      "lr": 4.486444199284386e-05,
      "elapsed_s": 2708.3243913650513,
      "loss": 3.4495468139648438,
      "hidden": 0.21626786515116692,
      "layer": 0.10042501240968704,
      "mlm": 1.7141005098819733,
      "distill": 1.2526980340480804,
      "teacher_acc": 0.7053470015525818
    },
    {
      "step": 3651,
      "lr": 4.4833833507280884e-05,
      "elapsed_s": 2715.731525659561,
      "loss": 3.2745672464370728,
      "hidden": 0.21886007115244865,
      "layer": 0.09998699836432934,
      "mlm": 1.607414722442627,
      "distill": 1.1794389188289642,
      "teacher_acc": 0.7276763916015625
    },
    {
      "step": 3661,
      "lr": 4.480314458449555e-05,
      "elapsed_s": 2723.1396899223328,
      "loss": 3.224293828010559,
      "hidden": 0.21279019862413406,
      "layer": 0.09969703480601311,
      "mlm": 1.5308388769626617,
      "distill": 1.218026041984558,
      "teacher_acc": 0.7359865456819534
    },
    {
      "step": 3671,
      "lr": 4.4772375348948514e-05,
      "elapsed_s": 2730.5405235290527,
      "loss": 3.4303205609321594,
      "hidden": 0.22217024862766266,
      "layer": 0.10104978643357754,
      "mlm": 1.6885684728622437,
      "distill": 1.2468867003917694,
      "teacher_acc": 0.7087351679801941
    },
    {
      "step": 3681,
      "lr": 4.4741525925426125e-05,
      "elapsed_s": 2737.9718265533447,
      "loss": 3.4497135281562805,
      "hidden": 0.2169898897409439,
      "layer": 0.10202844627201557,
      "mlm": 1.7016971409320831,
      "distill": 1.2630224227905273,
      "teacher_acc": 0.7157750129699707
    },
    {
      "step": 3691,
      "lr": 4.471059643903995e-05,
      "elapsed_s": 2745.4064025878906,
      "loss": 3.4879127740859985,
      "hidden": 0.2210364192724228,
      "layer": 0.10075599141418934,
      "mlm": 1.7257213592529297,
      "distill": 1.269740641117096,
      "teacher_acc": 0.7104261517524719
    },
    {
      "step": 3701,
      "lr": 4.4679587015226253e-05,
      "elapsed_s": 2752.8446593284607,
      "loss": 3.317773759365082,
      "hidden": 0.21198050305247307,
      "layer": 0.0990347359329462,
      "mlm": 1.612403392791748,
      "distill": 1.2318920195102692,
      "teacher_acc": 0.7241254895925522
    },
    {
      "step": 3711,
      "lr": 4.464849777974549e-05,
      "elapsed_s": 2760.2552025318146,
      "loss": 3.5187310576438904,
      "hidden": 0.22815653309226036,
      "layer": 0.10379868559539318,
      "mlm": 1.6347263753414154,
      "distill": 1.3757922053337097,
      "teacher_acc": 0.7245372980833054
    },
    {
      "step": 3721,
      "lr": 4.46173288586818e-05,
      "elapsed_s": 2767.6529121398926,
      "loss": 3.5593345761299133,
      "hidden": 0.22435739636421204,
      "layer": 0.10117137804627419,
      "mlm": 1.7613835334777832,
      "distill": 1.2986505031585693,
      "teacher_acc": 0.7059041857719421
    },
    {
      "step": 3731,
      "lr": 4.458608037844248e-05,
      "elapsed_s": 2775.083307981491,
      "loss": 3.3301835656166077,
      "hidden": 0.21828244253993034,
      "layer": 0.09901604987680912,
      "mlm": 1.6080121397972107,
      "distill": 1.2360985279083252,
      "teacher_acc": 0.7235274761915207
    },
    {
      "step": 3741,
      "lr": 4.455475246575749e-05,
      "elapsed_s": 2782.4781732559204,
      "loss": 3.484762966632843,
      "hidden": 0.2183942198753357,
      "layer": 0.10151520185172558,
      "mlm": 1.6876263320446014,
      "distill": 1.3095906376838684,
      "teacher_acc": 0.7111721634864807
    },
    {
      "step": 3751,
      "lr": 4.452334524767894e-05,
      "elapsed_s": 2789.8880689144135,
      "loss": 3.4765034317970276,
      "hidden": 0.22331636399030685,
      "layer": 0.10142433270812035,
      "mlm": 1.685680091381073,
      "distill": 1.2934783697128296,
      "teacher_acc": 0.7113285511732101
    },
    {
      "step": 3761,
      "lr": 4.449185885158056e-05,
      "elapsed_s": 2797.295140504837,
      "loss": 3.202395796775818,
      "hidden": 0.21051722019910812,
      "layer": 0.09814063087105751,
      "mlm": 1.5432848930358887,
      "distill": 1.189006119966507,
      "teacher_acc": 0.7244785130023956
    },
    {
      "step": 3771,
      "lr": 4.4460293405157196e-05,
      "elapsed_s": 2804.7213699817657,
      "loss": 3.422004997730255,
      "hidden": 0.215611781924963,
      "layer": 0.09998084418475628,
      "mlm": 1.6407859027385712,
      "distill": 1.3000050485134125,
      "teacher_acc": 0.7199901938438416
    },
    {
      "step": 3781,
      "lr": 4.442864903642428e-05,
      "elapsed_s": 2812.1592061519623,
      "loss": 3.3409595489501953,
      "hidden": 0.22222664579749107,
      "layer": 0.0999057237058878,
      "mlm": 1.5946301817893982,
      "distill": 1.2519233226776123,
      "teacher_acc": 0.7306882739067078
    },
    {
      "step": 3791,
      "lr": 4.439692587371731e-05,
      "elapsed_s": 2819.553843021393,
      "loss": 3.421941637992859,
      "hidden": 0.2181464582681656,
      "layer": 0.10009659826755524,
      "mlm": 1.6639005243778229,
      "distill": 1.2716999351978302,
      "teacher_acc": 0.7110802233219147
    },
    {
      "step": 3801,
      "lr": 4.436512404569136e-05,
      "elapsed_s": 2827.0108137130737,
      "loss": 3.4838470220565796,
      "hidden": 0.21770381927490234,
      "layer": 0.09972613863646984,
      "mlm": 1.6634467244148254,
      "distill": 1.3351296782493591,
      "teacher_acc": 0.7233183681964874
    },
    {
      "step": 3811,
      "lr": 4.4333243681320535e-05,
      "elapsed_s": 2834.431905269623,
      "loss": 3.407822072505951,
      "hidden": 0.2114402912557125,
      "layer": 0.09894207864999771,
      "mlm": 1.6837082207202911,
      "distill": 1.251762181520462,
      "teacher_acc": 0.7116187363862991
    },
    {
      "step": 3821,
      "lr": 4.430128490989742e-05,
      "elapsed_s": 2841.861021757126,
      "loss": 3.3526212573051453,
      "hidden": 0.2148815244436264,
      "layer": 0.09962364286184311,
      "mlm": 1.6373966038227081,
      "distill": 1.2356497943401337,
      "teacher_acc": 0.7233918309211731
    },
    {
      "step": 3831,
      "lr": 4.4269247861032614e-05,
      "elapsed_s": 2849.265626192093,
      "loss": 3.3154091238975525,
      "hidden": 0.20902447029948235,
      "layer": 0.09895804896950722,
      "mlm": 1.5924485623836517,
      "distill": 1.255432665348053,
      "teacher_acc": 0.728063628077507
    },
    {
      "step": 3841,
      "lr": 4.4237132664654154e-05,
      "elapsed_s": 2856.6546018123627,
      "loss": 3.248935580253601,
      "hidden": 0.2174443043768406,
      "layer": 0.0982291791588068,
      "mlm": 1.5518204867839813,
      "distill": 1.2131119668483734,
      "teacher_acc": 0.7329970300197601
    },
    {
      "step": 3851,
      "lr": 4.420493945100702e-05,
      "elapsed_s": 2864.039132118225,
      "loss": 3.241308331489563,
      "hidden": 0.21167390048503876,
      "layer": 0.09893447160720825,
      "mlm": 1.586848646402359,
      "distill": 1.1816445589065552,
      "teacher_acc": 0.7188943326473236
    },
    {
      "step": 3861,
      "lr": 4.417266835065259e-05,
      "elapsed_s": 2871.4359402656555,
      "loss": 3.520313799381256,
      "hidden": 0.22187251970171928,
      "layer": 0.10208077915012836,
      "mlm": 1.643601268529892,
      "distill": 1.3819271326065063,
      "teacher_acc": 0.7293011099100113
    },
    {
      "step": 3871,
      "lr": 4.414031949446812e-05,
      "elapsed_s": 2878.8153507709503,
      "loss": 3.3206436038017273,
      "hidden": 0.21320131793618202,
      "layer": 0.09920357540249825,
      "mlm": 1.609454721212387,
      "distill": 1.2351844012737274,
      "teacher_acc": 0.7314797937870026
    },
    {
      "step": 3881,
      "lr": 4.410789301364621e-05,
      "elapsed_s": 2886.2138121128082,
      "loss": 3.3598875403404236,
      "hidden": 0.20989752188324928,
      "layer": 0.10004802420735359,
      "mlm": 1.7030329406261444,
      "distill": 1.1870355904102325,
      "teacher_acc": 0.7016808539628983
    },
    {
      "step": 3891,
      "lr": 4.407538903969427e-05,
      "elapsed_s": 2893.605315923691,
      "loss": 3.3626590967178345,
      "hidden": 0.2143176905810833,
      "layer": 0.09860510751605034,
      "mlm": 1.6953279674053192,
      "distill": 1.1893931925296783,
      "teacher_acc": 0.7046426087617874
    },
    {
      "step": 3901,
      "lr": 4.404280770443398e-05,
      "elapsed_s": 2900.9553837776184,
      "loss": 3.2994500398635864,
      "hidden": 0.2169826552271843,
      "layer": 0.0994649063795805,
      "mlm": 1.5329503417015076,
      "distill": 1.2828018963336945,
      "teacher_acc": 0.7457766532897949
    },
    {
      "step": 3911,
      "lr": 4.401014914000078e-05,
      "elapsed_s": 2908.353337287903,
      "loss": 3.252126634120941,
      "hidden": 0.2164294384419918,
      "layer": 0.099966274574399,
      "mlm": 1.5539894998073578,
      "distill": 1.2152950465679169,
      "teacher_acc": 0.7420509904623032
    },
    {
      "step": 3921,
      "lr": 4.397741347884329e-05,
      "elapsed_s": 2915.753561258316,
      "loss": 3.4226454496383667,
      "hidden": 0.21871980279684067,
      "layer": 0.0997576005756855,
      "mlm": 1.6629964411258698,
      "distill": 1.2723305225372314,
      "teacher_acc": 0.713737279176712
    },
    {
      "step": 3931,
      "lr": 4.394460085372283e-05,
      "elapsed_s": 2923.082320690155,
      "loss": 3.3782944679260254,
      "hidden": 0.21189770475029945,
      "layer": 0.09814956411719322,
      "mlm": 1.6856461465358734,
      "distill": 1.2197779417037964,
      "teacher_acc": 0.7093459367752075
    },
    {
      "step": 3941,
      "lr": 4.391171139771284e-05,
      "elapsed_s": 2930.456774711609,
      "loss": 3.383235216140747,
      "hidden": 0.22015303745865822,
      "layer": 0.10052507556974888,
      "mlm": 1.6186807751655579,
      "distill": 1.2739859223365784,
      "teacher_acc": 0.7216889560222626
    },
    {
      "step": 3951,
      "lr": 4.3878745244198345e-05,
      "elapsed_s": 2937.8353567123413,
      "loss": 3.308003544807434,
      "hidden": 0.20746897533535957,
      "layer": 0.0978346262127161,
      "mlm": 1.6008215546607971,
      "distill": 1.2433266639709473,
      "teacher_acc": 0.7288355976343155
    },
    {
      "step": 3961,
      "lr": 4.384570252687542e-05,
      "elapsed_s": 2945.21405005455,
      "loss": 3.355210840702057,
      "hidden": 0.21652409434318542,
      "layer": 0.10084827803075314,
      "mlm": 1.586410015821457,
      "distill": 1.2853285372257233,
      "teacher_acc": 0.7303088307380676
    },
    {
      "step": 3971,
      "lr": 4.381258337975067e-05,
      "elapsed_s": 2952.630339384079,
      "loss": 3.3217382431030273,
      "hidden": 0.22013549134135246,
      "layer": 0.10000970214605331,
      "mlm": 1.5553718507289886,
      "distill": 1.2760905921459198,
      "teacher_acc": 0.7345653772354126
    },
    {
      "step": 3981,
      "lr": 4.377938793714064e-05,
      "elapsed_s": 2959.989623785019,
      "loss": 3.299283027648926,
      "hidden": 0.2244243435561657,
      "layer": 0.10162552073597908,
      "mlm": 1.463464081287384,
      "distill": 1.3361575305461884,
      "teacher_acc": 0.7579145282506943
    },
    {
      "step": 3991,
      "lr": 4.374611633367131e-05,
      "elapsed_s": 2967.376202583313,
      "loss": 3.354050636291504,
      "hidden": 0.21034714207053185,
      "layer": 0.09840697795152664,
      "mlm": 1.5843153893947601,
      "distill": 1.2998374700546265,
      "teacher_acc": 0.7269056886434555
    },
    {
      "step": 4001,
      "lr": 4.371276870427753e-05,
      "elapsed_s": 2976.8022191524506,
      "loss": 3.4065189957618713,
      "hidden": 0.21879760175943375,
      "layer": 0.10035175643861294,
      "mlm": 1.6638438999652863,
      "distill": 1.2549039721488953,
      "teacher_acc": 0.7246548235416412
    },
    {
      "step": 4011,
      "lr": 4.367934518420248e-05,
      "elapsed_s": 2984.1852300167084,
      "loss": 3.2665358781814575,
      "hidden": 0.21697533503174782,
      "layer": 0.09944034181535244,
      "mlm": 1.5963473916053772,
      "distill": 1.1865175664424896,
      "teacher_acc": 0.7277026772499084
    },
    {
      "step": 4021,
      "lr": 4.364584590899711e-05,
      "elapsed_s": 2991.5832533836365,
      "loss": 3.410503387451172,
      "hidden": 0.21479737758636475,
      "layer": 0.09951872564852238,
      "mlm": 1.6750191450119019,
      "distill": 1.2561302185058594,
      "teacher_acc": 0.720388725399971
    },
    {
      "step": 4031,
      "lr": 4.361227101451961e-05,
      "elapsed_s": 2999.0009472370148,
      "loss": 3.312085449695587,
      "hidden": 0.21299812942743301,
      "layer": 0.09873082488775253,
      "mlm": 1.61012402176857,
      "distill": 1.2265998125076294,
      "teacher_acc": 0.7221582680940628
    },
    {
      "step": 4041,
      "lr": 4.357862063693486e-05,
      "elapsed_s": 3006.4044857025146,
      "loss": 3.223383128643036,
      "hidden": 0.21718934550881386,
      "layer": 0.09884579107165337,
      "mlm": 1.575249046087265,
      "distill": 1.1643324792385101,
      "teacher_acc": 0.728322833776474
    },
    {
      "step": 4051,
      "lr": 4.354489491271383e-05,
      "elapsed_s": 3013.804839849472,
      "loss": 3.3336254358291626,
      "hidden": 0.21119142323732376,
      "layer": 0.09988928958773613,
      "mlm": 1.6594502627849579,
      "distill": 1.2018478214740753,
      "teacher_acc": 0.7138727158308029
    },
    {
      "step": 4061,
      "lr": 4.351109397863309e-05,
      "elapsed_s": 3021.226651906967,
      "loss": 3.4135611057281494,
      "hidden": 0.21501441299915314,
      "layer": 0.09955957159399986,
      "mlm": 1.68215212225914,
      "distill": 1.2516002655029297,
      "teacher_acc": 0.7062650769948959
    },
    {
      "step": 4071,
      "lr": 4.347721797177421e-05,
      "elapsed_s": 3028.6435391902924,
      "loss": 3.3159769773483276,
      "hidden": 0.2130233719944954,
      "layer": 0.0983841959387064,
      "mlm": 1.632628321647644,
      "distill": 1.2081098556518555,
      "teacher_acc": 0.7096691131591797
    },
    {
      "step": 4081,
      "lr": 4.344326702952326e-05,
      "elapsed_s": 3036.04949426651,
      "loss": 3.4724360704421997,
      "hidden": 0.21528591215610504,
      "layer": 0.09992446936666965,
      "mlm": 1.6721459925174713,
      "distill": 1.3197560608386993,
      "teacher_acc": 0.713673546910286
    },
    {
      "step": 4091,
      "lr": 4.3409241289570156e-05,
      "elapsed_s": 3043.4487726688385,
      "loss": 3.324856400489807,
      "hidden": 0.21483539789915085,
      "layer": 0.10057331807911396,
      "mlm": 1.5940937995910645,
      "distill": 1.2508050501346588,
      "teacher_acc": 0.7296435534954071
    },
    {
      "step": 4101,
      "lr": 4.337514088990822e-05,
      "elapsed_s": 3050.8427596092224,
      "loss": 3.2056392431259155,
      "hidden": 0.20962658524513245,
      "layer": 0.09801270067691803,
      "mlm": 1.5612215399742126,
      "distill": 1.1761581897735596,
      "teacher_acc": 0.7227577120065689
    },
    {
      "step": 4111,
      "lr": 4.3340965968833515e-05,
      "elapsed_s": 3058.261877298355,
      "loss": 3.2095713019371033,
      "hidden": 0.20669924467802048,
      "layer": 0.09755269065499306,
      "mlm": 1.5349131524562836,
      "distill": 1.2124833464622498,
      "teacher_acc": 0.7381237745285034
    },
    {
      "step": 4121,
      "lr": 4.3306716664944344e-05,
      "elapsed_s": 3065.653860092163,
      "loss": 3.2636516094207764,
      "hidden": 0.21468712389469147,
      "layer": 0.09911414608359337,
      "mlm": 1.5315880477428436,
      "distill": 1.2531323432922363,
      "teacher_acc": 0.7388000041246414
    },
    {
      "step": 4131,
      "lr": 4.327239311714068e-05,
      "elapsed_s": 3073.040103197098,
      "loss": 3.3153828382492065,
      "hidden": 0.2124047763645649,
      "layer": 0.09867933206260204,
      "mlm": 1.5492456257343292,
      "distill": 1.2919879853725433,
      "teacher_acc": 0.743598997592926
    },
    {
      "step": 4141,
      "lr": 4.323799546462361e-05,
      "elapsed_s": 3080.4622464179993,
      "loss": 3.2514199018478394,
      "hidden": 0.20567770302295685,
      "layer": 0.09758830443024635,
      "mlm": 1.583678275346756,
      "distill": 1.2075920403003693,
      "teacher_acc": 0.7260519862174988
    },
    {
      "step": 4151,
      "lr": 4.320352384689471e-05,
      "elapsed_s": 3087.8774950504303,
      "loss": 3.3389520049095154,
      "hidden": 0.21395226940512657,
      "layer": 0.09946276620030403,
      "mlm": 1.631710559129715,
      "distill": 1.229605495929718,
      "teacher_acc": 0.7242700904607773
    },
    {
      "step": 4161,
      "lr": 4.316897840375558e-05,
      "elapsed_s": 3095.2917170524597,
      "loss": 3.31244295835495,
      "hidden": 0.21337440237402916,
      "layer": 0.09964515455067158,
      "mlm": 1.5788481831550598,
      "distill": 1.2570233345031738,
      "teacher_acc": 0.7282462269067764
    },
    {
      "step": 4171,
      "lr": 4.313435927530719e-05,
      "elapsed_s": 3102.697306394577,
      "loss": 3.368472933769226,
      "hidden": 0.21270372718572617,
      "layer": 0.09802959859371185,
      "mlm": 1.6909606754779816,
      "distill": 1.203089952468872,
      "teacher_acc": 0.6991153061389923
    },
    {
      "step": 4181,
      "lr": 4.309966660194936e-05,
      "elapsed_s": 3110.094302415848,
      "loss": 3.403820812702179,
      "hidden": 0.21656644716858864,
      "layer": 0.09976123832166195,
      "mlm": 1.6377077102661133,
      "distill": 1.2830996215343475,
      "teacher_acc": 0.7238743305206299
    },
    {
      "step": 4191,
      "lr": 4.3064900524380156e-05,
      "elapsed_s": 3117.5210263729095,
      "loss": 3.3575798869132996,
      "hidden": 0.21645012125372887,
      "layer": 0.0990385003387928,
      "mlm": 1.6342847049236298,
      "distill": 1.240875780582428,
      "teacher_acc": 0.7318895757198334
    },
    {
      "step": 4201,
      "lr": 4.303006118359537e-05,
      "elapsed_s": 3124.916850090027,
      "loss": 3.2713816165924072,
      "hidden": 0.20467019826173782,
      "layer": 0.09689895249903202,
      "mlm": 1.6402555704116821,
      "distill": 1.1733361780643463,
      "teacher_acc": 0.7169571220874786
    },
    {
      "step": 4211,
      "lr": 4.299514872088788e-05,
      "elapsed_s": 3132.347605228424,
      "loss": 3.3704922199249268,
      "hidden": 0.21299269422888756,
      "layer": 0.09935298934578896,
      "mlm": 1.6011030972003937,
      "distill": 1.2937273383140564,
      "teacher_acc": 0.7330595850944519
    },
    {
      "step": 4221,
      "lr": 4.2960163277847144e-05,
      "elapsed_s": 3139.744080066681,
      "loss": 3.4132604002952576,
      "hidden": 0.22094204276800156,
      "layer": 0.101207559928298,
      "mlm": 1.6728207468986511,
      "distill": 1.247951716184616,
      "teacher_acc": 0.7104651629924774
    },
    {
      "step": 4231,
      "lr": 4.292510499635859e-05,
      "elapsed_s": 3147.1650512218475,
      "loss": 3.318944990634918,
      "hidden": 0.2157551795244217,
      "layer": 0.09840540960431099,
      "mlm": 1.6330899000167847,
      "distill": 1.2051419615745544,
      "teacher_acc": 0.7182030230760574
    },
    {
      "step": 4241,
      "lr": 4.288997401860303e-05,
      "elapsed_s": 3154.5866589546204,
      "loss": 3.3167114853858948,
      "hidden": 0.2132684588432312,
      "layer": 0.09918131120502949,
      "mlm": 1.592810332775116,
      "distill": 1.2477736175060272,
      "teacher_acc": 0.7278425097465515
    },
    {
      "step": 4251,
      "lr": 4.2854770487056114e-05,
      "elapsed_s": 3161.968564271927,
      "loss": 3.3776387572288513,
      "hidden": 0.20509525388479233,
      "layer": 0.09673847258090973,
      "mlm": 1.7136280834674835,
      "distill": 1.2054509222507477,
      "teacher_acc": 0.6952854245901108
    },
    {
      "step": 4261,
      "lr": 4.281949454448775e-05,
      "elapsed_s": 3169.3572041988373,
      "loss": 3.348029613494873,
      "hidden": 0.21489091217517853,
      "layer": 0.09820382855832577,
      "mlm": 1.64179128408432,
      "distill": 1.2273545265197754,
      "teacher_acc": 0.7184735685586929
    },
    {
      "step": 4271,
      "lr": 4.2784146333961484e-05,
      "elapsed_s": 3176.742824792862,
      "loss": 3.346528470516205,
      "hidden": 0.2108408696949482,
      "layer": 0.09788176417350769,
      "mlm": 1.6464946866035461,
      "distill": 1.2294111847877502,
      "teacher_acc": 0.7197493314743042
    },
    {
      "step": 4281,
      "lr": 4.274872599883396e-05,
      "elapsed_s": 3184.1224415302277,
      "loss": 3.3095152974128723,
      "hidden": 0.22172212228178978,
      "layer": 0.10062669776380062,
      "mlm": 1.5060188472270966,
      "distill": 1.3097388744354248,
      "teacher_acc": 0.756232351064682
    },
    {
      "step": 4291,
      "lr": 4.2713233682754336e-05,
      "elapsed_s": 3191.491533279419,
      "loss": 3.3153995871543884,
      "hidden": 0.2121831253170967,
      "layer": 0.09842216409742832,
      "mlm": 1.6252050697803497,
      "distill": 1.2166172564029694,
      "teacher_acc": 0.71697136759758
    },
    {
      "step": 4301,
      "lr": 4.267766952966369e-05,
      "elapsed_s": 3198.8597388267517,
      "loss": 3.2286808490753174,
      "hidden": 0.21142106503248215,
      "layer": 0.09760724380612373,
      "mlm": 1.4907747507095337,
      "distill": 1.2662602663040161,
      "teacher_acc": 0.7473519891500473
    },
    {
      "step": 4311,
      "lr": 4.264203368379444e-05,
      "elapsed_s": 3206.2546882629395,
      "loss": 3.309619665145874,
      "hidden": 0.21119853109121323,
      "layer": 0.09820627421140671,
      "mlm": 1.6032266020774841,
      "distill": 1.2348929047584534,
      "teacher_acc": 0.725422590970993
    },
    {
      "step": 4321,
      "lr": 4.260632628966974e-05,
      "elapsed_s": 3213.632351875305,
      "loss": 3.247652053833008,
      "hidden": 0.20677784457802773,
      "layer": 0.09694913029670715,
      "mlm": 1.6099347472190857,
      "distill": 1.175687164068222,
      "teacher_acc": 0.7078178822994232
    },
    {
      "step": 4331,
      "lr": 4.257054749210293e-05,
      "elapsed_s": 3221.0118060112,
      "loss": 3.382774829864502,
      "hidden": 0.2139727734029293,
      "layer": 0.09984396398067474,
      "mlm": 1.6403425931930542,
      "distill": 1.264564722776413,
      "teacher_acc": 0.7251731604337692
    },
    {
      "step": 4341,
      "lr": 4.253469743619694e-05,
      "elapsed_s": 3228.431436777115,
      "loss": 3.1757484078407288,
      "hidden": 0.20999810844659805,
      "layer": 0.0969794224947691,
      "mlm": 1.566969394683838,
      "distill": 1.140293151140213,
      "teacher_acc": 0.7240430414676666
    },
    {
      "step": 4351,
      "lr": 4.2498776267343664e-05,
      "elapsed_s": 3235.809595346451,
      "loss": 3.327117443084717,
      "hidden": 0.20848198980093002,
      "layer": 0.09885729104280472,
      "mlm": 1.666463017463684,
      "distill": 1.194261759519577,
      "teacher_acc": 0.707717552781105
    },
    {
      "step": 4361,
      "lr": 4.2462784131223434e-05,
      "elapsed_s": 3243.16907286644,
      "loss": 3.245089888572693,
      "hidden": 0.20396515354514122,
      "layer": 0.09602841548621655,
      "mlm": 1.6105963289737701,
      "distill": 1.1785490214824677,
      "teacher_acc": 0.7206247895956039
    },
    {
      "step": 4371,
      "lr": 4.242672117380434e-05,
      "elapsed_s": 3250.5591628551483,
      "loss": 3.202542722225189,
      "hidden": 0.20455225557088852,
      "layer": 0.09619609825313091,
      "mlm": 1.5679444670677185,
      "distill": 1.1773956716060638,
      "teacher_acc": 0.729967474937439
    },
    {
      "step": 4381,
      "lr": 4.239058754134175e-05,
      "elapsed_s": 3257.9884610176086,
      "loss": 3.3390700817108154,
      "hidden": 0.2143702432513237,
      "layer": 0.09926500730216503,
      "mlm": 1.6220947802066803,
      "distill": 1.2386023998260498,
      "teacher_acc": 0.7230208814144135
    },
    {
      "step": 4391,
      "lr": 4.235438338037764e-05,
      "elapsed_s": 3265.413781642914,
      "loss": 3.19665265083313,
      "hidden": 0.20591288432478905,
      "layer": 0.09592483565211296,
      "mlm": 1.5535687804222107,
      "distill": 1.1832956373691559,
      "teacher_acc": 0.7293404340744019
    },
    {
      "step": 4401,
      "lr": 4.231810883773999e-05,
      "elapsed_s": 3272.8304164409637,
      "loss": 3.3925610184669495,
      "hidden": 0.20864951238036156,
      "layer": 0.09717600978910923,
      "mlm": 1.7057346105575562,
      "distill": 1.2209393978118896,
      "teacher_acc": 0.7065473347902298
    },
    {
      "step": 4411,
      "lr": 4.228176406054225e-05,
      "elapsed_s": 3280.2234992980957,
      "loss": 3.154885172843933,
      "hidden": 0.20931454375386238,
      "layer": 0.09889213182032108,
      "mlm": 1.5483828485012054,
      "distill": 1.1384271085262299,
      "teacher_acc": 0.7219444215297699
    },
    {
      "step": 4421,
      "lr": 4.224534919618269e-05,
      "elapsed_s": 3287.612589120865,
      "loss": 3.316923677921295,
      "hidden": 0.2083878554403782,
      "layer": 0.09787110239267349,
      "mlm": 1.6521506011486053,
      "distill": 1.1990618109703064,
      "teacher_acc": 0.7116932272911072
    },
    {
      "step": 4431,
      "lr": 4.220886439234385e-05,
      "elapsed_s": 3294.998076915741,
      "loss": 3.322017788887024,
      "hidden": 0.20995421335101128,
      "layer": 0.09641577675938606,
      "mlm": 1.6472203135490417,
      "distill": 1.206681102514267,
      "teacher_acc": 0.7126714438199997
    },
    {
      "step": 4441,
      "lr": 4.217230979699188e-05,
      "elapsed_s": 3302.400155067444,
      "loss": 3.2233042120933533,
      "hidden": 0.21804075688123703,
      "layer": 0.09952028468251228,
      "mlm": 1.4834250807762146,
      "distill": 1.2540375292301178,
      "teacher_acc": 0.7488970756530762
    },
    {
      "step": 4451,
      "lr": 4.2135685558375994e-05,
      "elapsed_s": 3309.8062233924866,
      "loss": 3.243834614753723,
      "hidden": 0.21329382807016373,
      "layer": 0.09946012683212757,
      "mlm": 1.5552393794059753,
      "distill": 1.2122775614261627,
      "teacher_acc": 0.7356916218996048
    },
    {
      "step": 4461,
      "lr": 4.2098991825027844e-05,
      "elapsed_s": 3317.1986808776855,
      "loss": 3.1683651208877563,
      "hidden": 0.2039036601781845,
      "layer": 0.09574254602193832,
      "mlm": 1.5798918902873993,
      "distill": 1.1327946782112122,
      "teacher_acc": 0.7229364961385727
    },
    {
      "step": 4471,
      "lr": 4.206222874576091e-05,
      "elapsed_s": 3324.6185138225555,
      "loss": 3.5012460947036743,
      "hidden": 0.21331216394901276,
      "layer": 0.09866055101156235,
      "mlm": 1.688195526599884,
      "distill": 1.337095856666565,
      "teacher_acc": 0.7197980880737305
    },
    {
      "step": 4481,
      "lr": 4.202539646966993e-05,
      "elapsed_s": 3332.0245263576508,
      "loss": 3.235841155052185,
      "hidden": 0.2092849314212799,
      "layer": 0.09713458828628063,
      "mlm": 1.5866239368915558,
      "distill": 1.182080090045929,
      "teacher_acc": 0.7247676998376846
    },
    {
      "step": 4491,
      "lr": 4.198849514613026e-05,
      "elapsed_s": 3339.364906311035,
      "loss": 3.348326861858368,
      "hidden": 0.20757995173335075,
      "layer": 0.09760897234082222,
      "mlm": 1.7007623314857483,
      "distill": 1.1836000978946686,
      "teacher_acc": 0.7006926983594894
    },
    {
      "step": 4501,
      "lr": 4.195152492479727e-05,
      "elapsed_s": 3346.719263792038,
      "loss": 3.366444408893585,
      "hidden": 0.21191087737679482,
      "layer": 0.09849878028035164,
      "mlm": 1.677823781967163,
      "distill": 1.2155494093894958,
      "teacher_acc": 0.709282785654068
    },
    {
      "step": 4511,
      "lr": 4.1914485955605785e-05,
      "elapsed_s": 3354.1317234039307,
      "loss": 3.2004151940345764,
      "hidden": 0.2059791535139084,
      "layer": 0.09623594023287296,
      "mlm": 1.562754601240158,
      "distill": 1.1775842905044556,
      "teacher_acc": 0.7337772995233536
    },
    {
      "step": 4521,
      "lr": 4.1877378388769406e-05,
      "elapsed_s": 3361.514974832535,
      "loss": 3.3409120440483093,
      "hidden": 0.21115676686167717,
      "layer": 0.09899488277733326,
      "mlm": 1.678274929523468,
      "distill": 1.190826028585434,
      "teacher_acc": 0.7062052488327026
    },
    {
      "step": 4531,
      "lr": 4.1840202374779945e-05,
      "elapsed_s": 3368.8570654392242,
      "loss": 3.3012958765029907,
      "hidden": 0.21341875568032265,
      "layer": 0.09695042110979557,
      "mlm": 1.598259836435318,
      "distill": 1.227723240852356,
      "teacher_acc": 0.729526549577713
    },
    {
      "step": 4541,
      "lr": 4.180295806440682e-05,
      "elapsed_s": 3376.2064316272736,
      "loss": 3.30934602022171,
      "hidden": 0.2079078033566475,
      "layer": 0.0985367875546217,
      "mlm": 1.6045744121074677,
      "distill": 1.239687591791153,
      "teacher_acc": 0.7236827909946442
    },
    {
      "step": 4551,
      "lr": 4.1765645608696404e-05,
      "elapsed_s": 3383.593672990799,
      "loss": 3.3828004002571106,
      "hidden": 0.2159968875348568,
      "layer": 0.09976777620613575,
      "mlm": 1.702335238456726,
      "distill": 1.198587417602539,
      "teacher_acc": 0.6992685049772263
    },
    {
      "step": 4561,
      "lr": 4.172826515897146e-05,
      "elapsed_s": 3391.024036169052,
      "loss": 3.331816077232361,
      "hidden": 0.20896141231060028,
      "layer": 0.0994015708565712,
      "mlm": 1.6255565583705902,
      "distill": 1.2386358678340912,
      "teacher_acc": 0.7311897575855255
    },
    {
      "step": 4571,
      "lr": 4.169081686683049e-05,
      "elapsed_s": 3398.3837625980377,
      "loss": 3.42166531085968,
      "hidden": 0.20937034115195274,
      "layer": 0.0973339956253767,
      "mlm": 1.6915410161018372,
      "distill": 1.2627165913581848,
      "teacher_acc": 0.7185308933258057
    },
    {
      "step": 4581,
      "lr": 4.165330088414714e-05,
      "elapsed_s": 3405.7707788944244,
      "loss": 3.2203798294067383,
      "hidden": 0.20206398516893387,
      "layer": 0.0966983325779438,
      "mlm": 1.6712635159492493,
      "distill": 1.0966392308473587,
      "teacher_acc": 0.6975877285003662
    },
    {
      "step": 4591,
      "lr": 4.1615717363069575e-05,
      "elapsed_s": 3413.1664209365845,
      "loss": 3.208263397216797,
      "hidden": 0.2110174223780632,
      "layer": 0.09827472269535065,
      "mlm": 1.5549829304218292,
      "distill": 1.1821082532405853,
      "teacher_acc": 0.7375730723142624
    },
    {
      "step": 4601,
      "lr": 4.157806645601988e-05,
      "elapsed_s": 3420.52996468544,
      "loss": 3.1552221179008484,
      "hidden": 0.2047625035047531,
      "layer": 0.09580486454069614,
      "mlm": 1.49265456199646,
      "distill": 1.205140084028244,
      "teacher_acc": 0.7411182075738907
    },
    {
      "step": 4611,
      "lr": 4.154034831569341e-05,
      "elapsed_s": 3427.913978099823,
      "loss": 3.264908015727997,
      "hidden": 0.20728349313139915,
      "layer": 0.09630329348146915,
      "mlm": 1.6494484841823578,
      "distill": 1.1527409255504608,
      "teacher_acc": 0.7079003751277924
    },
    {
      "step": 4621,
      "lr": 4.15025630950582e-05,
      "elapsed_s": 3435.289041996002,
      "loss": 3.2947943806648254,
      "hidden": 0.21196464449167252,
      "layer": 0.09842529334127903,
      "mlm": 1.5996275842189789,
      "distill": 1.222024917602539,
      "teacher_acc": 0.7145718932151794
    },
    {
      "step": 4631,
      "lr": 4.1464710947354313e-05,
      "elapsed_s": 3442.657971382141,
      "loss": 3.199162185192108,
      "hidden": 0.2088560238480568,
      "layer": 0.09814333729445934,
      "mlm": 1.5912317633628845,
      "distill": 1.1411466300487518,
      "teacher_acc": 0.719153419137001
    },
    {
      "step": 4641,
      "lr": 4.1426792026093276e-05,
      "elapsed_s": 3450.0308136940002,
      "loss": 3.2100967168807983,
      "hidden": 0.20065710693597794,
      "layer": 0.09514385275542736,
      "mlm": 1.6157848238945007,
      "distill": 1.1454257369041443,
      "teacher_acc": 0.7144080400466919
    },
    {
      "step": 4651,
      "lr": 4.138880648505738e-05,
      "elapsed_s": 3457.429240465164,
      "loss": 3.269571363925934,
      "hidden": 0.20207364857196808,
      "layer": 0.09576329216361046,
      "mlm": 1.6158208847045898,
      "distill": 1.2017216682434082,
      "teacher_acc": 0.72641821205616
    },
    {
      "step": 4661,
      "lr": 4.1350754478299114e-05,
      "elapsed_s": 3464.8490397930145,
      "loss": 3.3542093634605408,
      "hidden": 0.21145424991846085,
      "layer": 0.09811549261212349,
      "mlm": 1.651604026556015,
      "distill": 1.2306391596794128,
      "teacher_acc": 0.719676747918129
    },
    {
      "step": 4671,
      "lr": 4.131263616014053e-05,
      "elapsed_s": 3472.242677927017,
      "loss": 3.112748682498932,
      "hidden": 0.20154592767357826,
      "layer": 0.09408058784902096,
      "mlm": 1.5505365431308746,
      "distill": 1.112079918384552,
      "teacher_acc": 0.72315414249897
    },
    {
      "step": 4681,
      "lr": 4.12744516851726e-05,
      "elapsed_s": 3479.6217126846313,
      "loss": 3.318545162677765,
      "hidden": 0.20495014265179634,
      "layer": 0.09570681303739548,
      "mlm": 1.618401825428009,
      "distill": 1.2423896491527557,
      "teacher_acc": 0.7313784509897232
    },
    {
      "step": 4691,
      "lr": 4.123620120825459e-05,
      "elapsed_s": 3486.988903284073,
      "loss": 3.135640025138855,
      "hidden": 0.20741036906838417,
      "layer": 0.09641355089843273,
      "mlm": 1.5327167809009552,
      "distill": 1.1398956775665283,
      "teacher_acc": 0.7309237420558929
    },
    {
      "step": 4701,
      "lr": 4.1197884884513474e-05,
      "elapsed_s": 3494.3517067432404,
      "loss": 3.1240809559822083,
      "hidden": 0.20632804185152054,
      "layer": 0.0962603073567152,
      "mlm": 1.476097822189331,
      "distill": 1.1871967911720276,
      "teacher_acc": 0.7478890419006348
    },
    {
      "step": 4711,
      "lr": 4.115950286934322e-05,
      "elapsed_s": 3501.728793144226,
      "loss": 3.3512917160987854,
      "hidden": 0.20859292894601822,
      "layer": 0.09733176603913307,
      "mlm": 1.6412251889705658,
      "distill": 1.2442147731781006,
      "teacher_acc": 0.7244714945554733
    },
    {
      "step": 4721,
      "lr": 4.1121055318404264e-05,
      "elapsed_s": 3509.08344578743,
      "loss": 3.1968942284584045,
      "hidden": 0.20581138134002686,
      "layer": 0.09742253832519054,
      "mlm": 1.5619724094867706,
      "distill": 1.1745876967906952,
      "teacher_acc": 0.7246065437793732
    },
    {
      "step": 4731,
      "lr": 4.108254238762279e-05,
      "elapsed_s": 3516.442927837372,
      "loss": 3.299632728099823,
      "hidden": 0.20540426298975945,
      "layer": 0.09687220677733421,
      "mlm": 1.6597283482551575,
      "distill": 1.1806597709655762,
      "teacher_acc": 0.7085152864456177
    },
    {
      "step": 4741,
      "lr": 4.1043964233190146e-05,
      "elapsed_s": 3523.8042390346527,
      "loss": 3.2163124084472656,
      "hidden": 0.2063148468732834,
      "layer": 0.09597291797399521,
      "mlm": 1.5742062032222748,
      "distill": 1.1814900636672974,
      "teacher_acc": 0.7179129868745804
    },
    {
      "step": 4751,
      "lr": 4.1005321011562206e-05,
      "elapsed_s": 3531.179769515991,
      "loss": 3.218534469604492,
      "hidden": 0.20405955612659454,
      "layer": 0.0959672536700964,
      "mlm": 1.559091329574585,
      "distill": 1.2033403515815735,
      "teacher_acc": 0.7293764501810074
    },
    {
      "step": 4761,
      "lr": 4.09666128794587e-05,
      "elapsed_s": 3538.5659732818604,
      "loss": 3.308561682701111,
      "hidden": 0.21187366172671318,
      "layer": 0.09824804961681366,
      "mlm": 1.604141116142273,
      "distill": 1.231549233198166,
      "teacher_acc": 0.7184893935918808
    },
    {
      "step": 4771,
      "lr": 4.0927839993862644e-05,
      "elapsed_s": 3545.9776673316956,
      "loss": 3.3494094610214233,
      "hidden": 0.20454757660627365,
      "layer": 0.09713119082152843,
      "mlm": 1.6668068170547485,
      "distill": 1.2249418795108795,
      "teacher_acc": 0.7053567618131638
    },
    {
      "step": 4781,
      "lr": 4.088900251201964e-05,
      "elapsed_s": 3553.414164543152,
      "loss": 3.2787808775901794,
      "hidden": 0.21274873241782188,
      "layer": 0.09825598634779453,
      "mlm": 1.5920704901218414,
      "distill": 1.2120848894119263,
      "teacher_acc": 0.7195242494344711
    },
    {
      "step": 4791,
      "lr": 4.0850100591437256e-05,
      "elapsed_s": 3560.8053572177887,
      "loss": 3.307882845401764,
      "hidden": 0.2057039737701416,
      "layer": 0.09678362868726254,
      "mlm": 1.7040260136127472,
      "distill": 1.1440570503473282,
      "teacher_acc": 0.6977433264255524
    },
    {
      "step": 4801,
      "lr": 4.0811134389884433e-05,
      "elapsed_s": 3568.234459877014,
      "loss": 3.1556836366653442,
      "hidden": 0.20953970029950142,
      "layer": 0.09672900661826134,
      "mlm": 1.5131571590900421,
      "distill": 1.1750824749469757,
      "teacher_acc": 0.7338665872812271
    },
    {
      "step": 4811,
      "lr": 4.077210406539077e-05,
      "elapsed_s": 3575.6317596435547,
      "loss": 3.2032363414764404,
      "hidden": 0.20511845871806145,
      "layer": 0.09688124619424343,
      "mlm": 1.504835069179535,
      "distill": 1.2397237420082092,
      "teacher_acc": 0.7389679998159409
    },
    {
      "step": 4821,
      "lr": 4.073300977624594e-05,
      "elapsed_s": 3583.0569841861725,
      "loss": 3.2924527525901794,
      "hidden": 0.21002959087491035,
      "layer": 0.09711155295372009,
      "mlm": 1.5760244727134705,
      "distill": 1.2478132843971252,
      "teacher_acc": 0.7332588732242584
    },
    {
      "step": 4831,
      "lr": 4.069385168099902e-05,
      "elapsed_s": 3590.426653623581,
      "loss": 3.1348413228988647,
      "hidden": 0.19648142158985138,
      "layer": 0.09512872621417046,
      "mlm": 1.5534532070159912,
      "distill": 1.1408608853816986,
      "teacher_acc": 0.7301175147294998
    },
    {
      "step": 4841,
      "lr": 4.065462993845784e-05,
      "elapsed_s": 3597.8248150348663,
      "loss": 3.2645967602729797,
      "hidden": 0.20437493920326233,
      "layer": 0.09556300565600395,
      "mlm": 1.6643647849559784,
      "distill": 1.143700659275055,
      "teacher_acc": 0.7002401202917099
    },
    {
      "step": 4851,
      "lr": 4.061534470768841e-05,
      "elapsed_s": 3605.1821913719177,
      "loss": 3.262784004211426,
      "hidden": 0.20648889616131783,
      "layer": 0.09566755220293999,
      "mlm": 1.6383762657642365,
      "distill": 1.163596123456955,
      "teacher_acc": 0.7078545987606049
    },
    {
      "step": 4861,
      "lr": 4.057599614801415e-05,
      "elapsed_s": 3612.5551936626434,
      "loss": 3.3020431995391846,
      "hidden": 0.20443880930542946,
      "layer": 0.09519122540950775,
      "mlm": 1.6234206557273865,
      "distill": 1.2221492528915405,
      "teacher_acc": 0.7199424654245377
    },
    {
      "step": 4871,
      "lr": 4.053658441901536e-05,
      "elapsed_s": 3619.919929265976,
      "loss": 3.057672083377838,
      "hidden": 0.19774561002850533,
      "layer": 0.09378297254443169,
      "mlm": 1.4854837656021118,
      "distill": 1.1298055946826935,
      "teacher_acc": 0.737074613571167
    },
    {
      "step": 4881,
      "lr": 4.049710968052851e-05,
      "elapsed_s": 3627.2914140224457,
      "loss": 3.1329058408737183,
      "hidden": 0.20216650888323784,
      "layer": 0.09563377872109413,
      "mlm": 1.5161761939525604,
      "distill": 1.1645798087120056,
      "teacher_acc": 0.735722005367279
    },
    {
      "step": 4891,
      "lr": 4.045757209264562e-05,
      "elapsed_s": 3634.6763672828674,
      "loss": 3.173789620399475,
      "hidden": 0.20137259736657143,
      "layer": 0.09478520601987839,
      "mlm": 1.5647208988666534,
      "distill": 1.1589308828115463,
      "teacher_acc": 0.727718323469162
    },
    {
      "step": 4901,
      "lr": 4.0417971815713584e-05,
      "elapsed_s": 3642.0988109111786,
      "loss": 3.2118477821350098,
      "hidden": 0.2055814266204834,
      "layer": 0.09653116017580032,
      "mlm": 1.5730860233306885,
      "distill": 1.179333359003067,
      "teacher_acc": 0.7336676716804504
    },
    {
      "step": 4911,
      "lr": 4.0378309010333535e-05,
      "elapsed_s": 3649.510662317276,
      "loss": 3.2011208534240723,
      "hidden": 0.203938540071249,
      "layer": 0.09536782465875149,
      "mlm": 1.56002476811409,
      "distill": 1.185535043478012,
      "teacher_acc": 0.7288943380117416
    },
    {
      "step": 4921,
      "lr": 4.0338583837360225e-05,
      "elapsed_s": 3656.876801967621,
      "loss": 3.2207789421081543,
      "hidden": 0.19918324798345566,
      "layer": 0.09405398555099964,
      "mlm": 1.6348387598991394,
      "distill": 1.1405466794967651,
      "teacher_acc": 0.710679292678833
    },
    {
      "step": 4931,
      "lr": 4.02987964579013e-05,
      "elapsed_s": 3664.236213207245,
      "loss": 3.2520132660865784,
      "hidden": 0.20406421646475792,
      "layer": 0.0951437670737505,
      "mlm": 1.5828405916690826,
      "distill": 1.2134723961353302,
      "teacher_acc": 0.7262012660503387
    },
    {
      "step": 4941,
      "lr": 4.025894703331671e-05,
      "elapsed_s": 3671.626344203949,
      "loss": 3.105576276779175,
      "hidden": 0.20592112839221954,
      "layer": 0.09550800919532776,
      "mlm": 1.478015959262848,
      "distill": 1.1679640710353851,
      "teacher_acc": 0.7497824877500534
    },
    {
      "step": 4951,
      "lr": 4.021903572521802e-05,
      "elapsed_s": 3678.9959321022034,
      "loss": 3.399523437023163,
      "hidden": 0.21514524891972542,
      "layer": 0.10221908055245876,
      "mlm": 1.5978756248950958,
      "distill": 1.320247769355774,
      "teacher_acc": 0.7314503788948059
    },
    {
      "step": 4961,
      "lr": 4.0179062695467784e-05,
      "elapsed_s": 3686.3737268447876,
      "loss": 3.180026412010193,
      "hidden": 0.20108840242028236,
      "layer": 0.09561615996062756,
      "mlm": 1.559617966413498,
      "distill": 1.1704235970973969,
      "teacher_acc": 0.730669841170311
    },
    {
      "step": 4971,
      "lr": 4.013902810617885e-05,
      "elapsed_s": 3693.774680376053,
      "loss": 3.2683951258659363,
      "hidden": 0.20495279133319855,
      "layer": 0.09737017937004566,
      "mlm": 1.5766583383083344,
      "distill": 1.2331461906433105,
      "teacher_acc": 0.7283996194601059
    },
    {
      "step": 4981,
      "lr": 4.009893211971376e-05,
      "elapsed_s": 3701.1765642166138,
      "loss": 3.221259891986847,
      "hidden": 0.20638132467865944,
      "layer": 0.09570233710110188,
      "mlm": 1.5900909006595612,
      "distill": 1.1705550849437714,
      "teacher_acc": 0.7258387058973312
    },
    {
      "step": 4991,
      "lr": 4.005877489868401e-05,
      "elapsed_s": 3708.545308828354,
      "loss": 3.153154134750366,
      "hidden": 0.20396416634321213,
      "layer": 0.0966196246445179,
      "mlm": 1.5876067280769348,
      "distill": 1.1093092858791351,
      "teacher_acc": 0.7189137041568756
    },
    {
      "step": 5001,
      "lr": 4.001855660594948e-05,
      "elapsed_s": 3717.985068321228,
      "loss": 3.2355785369873047,
      "hidden": 0.20908984914422035,
      "layer": 0.09800935909152031,
      "mlm": 1.539255976676941,
      "distill": 1.2291381657123566,
      "teacher_acc": 0.7423324584960938
    },
    {
      "step": 5011,
      "lr": 3.997827740461768e-05,
      "elapsed_s": 3725.369317293167,
      "loss": 3.2152320742607117,
      "hidden": 0.2005365490913391,
      "layer": 0.09435624070465565,
      "mlm": 1.6410862803459167,
      "distill": 1.1258945763111115,
      "teacher_acc": 0.7072471529245377
    },
    {
      "step": 5021,
      "lr": 3.993793745804319e-05,
      "elapsed_s": 3732.7243218421936,
      "loss": 3.241732120513916,
      "hidden": 0.20025945454835892,
      "layer": 0.0962473712861538,
      "mlm": 1.5967066884040833,
      "distill": 1.1963827908039093,
      "teacher_acc": 0.7308148443698883
    },
    {
      "step": 5031,
      "lr": 3.98975369298269e-05,
      "elapsed_s": 3740.0999846458435,
      "loss": 3.230296552181244,
      "hidden": 0.2036784552037716,
      "layer": 0.0959779042750597,
      "mlm": 1.6101833879947662,
      "distill": 1.1647672355175018,
      "teacher_acc": 0.7241557389497757
    },
    {
      "step": 5041,
      "lr": 3.985707598381544e-05,
      "elapsed_s": 3747.4802165031433,
      "loss": 3.335447609424591,
      "hidden": 0.21229297295212746,
      "layer": 0.09786086902022362,
      "mlm": 1.6643494069576263,
      "distill": 1.1975817680358887,
      "teacher_acc": 0.7102034389972687
    },
    {
      "step": 5051,
      "lr": 3.981655478410043e-05,
      "elapsed_s": 3754.8813359737396,
      "loss": 3.2025309801101685,
      "hidden": 0.19948890432715416,
      "layer": 0.09439263865351677,
      "mlm": 1.6363706290721893,
      "distill": 1.119986206293106,
      "teacher_acc": 0.7115653604269028
    },
    {
      "step": 5061,
      "lr": 3.977597349501785e-05,
      "elapsed_s": 3762.268499135971,
      "loss": 3.1651260256767273,
      "hidden": 0.20650998502969742,
      "layer": 0.09630860015749931,
      "mlm": 1.5793797671794891,
      "distill": 1.124572068452835,
      "teacher_acc": 0.7204500436782837
    },
    {
      "step": 5071,
      "lr": 3.973533228114742e-05,
      "elapsed_s": 3769.679871559143,
      "loss": 3.111977994441986,
      "hidden": 0.19942068681120872,
      "layer": 0.09455664083361626,
      "mlm": 1.5150408446788788,
      "distill": 1.150817483663559,
      "teacher_acc": 0.7332603186368942
    },
    {
      "step": 5081,
      "lr": 3.969463130731183e-05,
      "elapsed_s": 3777.088123559952,
      "loss": 3.228224515914917,
      "hidden": 0.21238475292921066,
      "layer": 0.09742080047726631,
      "mlm": 1.5520416498184204,
      "distill": 1.202702909708023,
      "teacher_acc": 0.7264745682477951
    },
    {
      "step": 5091,
      "lr": 3.965387073857617e-05,
      "elapsed_s": 3784.4977946281433,
      "loss": 3.2209428548812866,
      "hidden": 0.20746779069304466,
      "layer": 0.09702244587242603,
      "mlm": 1.5500885844230652,
      "distill": 1.2074074447154999,
      "teacher_acc": 0.7321085929870605
    },
    {
      "step": 5101,
      "lr": 3.9613050740247224e-05,
      "elapsed_s": 3791.930071592331,
      "loss": 3.1127697229385376,
      "hidden": 0.1943652518093586,
      "layer": 0.09326641820371151,
      "mlm": 1.5846974849700928,
      "distill": 1.0927084982395172,
      "teacher_acc": 0.7157490998506546
    },
    {
      "step": 5111,
      "lr": 3.957217147787276e-05,
      "elapsed_s": 3799.361764907837,
      "loss": 3.160230040550232,
      "hidden": 0.19682473316788673,
      "layer": 0.09407675080001354,
      "mlm": 1.5076867640018463,
      "distill": 1.2118555009365082,
      "teacher_acc": 0.7399500012397766
    },
    {
      "step": 5121,
      "lr": 3.9531233117240916e-05,
      "elapsed_s": 3806.7531530857086,
      "loss": 3.265174627304077,
      "hidden": 0.2011437937617302,
      "layer": 0.0954657569527626,
      "mlm": 1.6025209724903107,
      "distill": 1.2126331329345703,
      "teacher_acc": 0.7237385213375092
    },
    {
      "step": 5131,
      "lr": 3.949023582437952e-05,
      "elapsed_s": 3814.1271891593933,
      "loss": 3.2590845227241516,
      "hidden": 0.20793559402227402,
      "layer": 0.09669565036892891,
      "mlm": 1.604582279920578,
      "distill": 1.1902831494808197,
      "teacher_acc": 0.7265826314687729
    },
    {
      "step": 5141,
      "lr": 3.9449179765555375e-05,
      "elapsed_s": 3821.4902596473694,
      "loss": 3.1402412056922913,
      "hidden": 0.1993517056107521,
      "layer": 0.09362811036407948,
      "mlm": 1.563025563955307,
      "distill": 1.1316982209682465,
      "teacher_acc": 0.7202557474374771
    },
    {
      "step": 5151,
      "lr": 3.940806510727364e-05,
      "elapsed_s": 3828.8709874153137,
      "loss": 3.1213550567626953,
      "hidden": 0.20117231830954552,
      "layer": 0.09498413279652596,
      "mlm": 1.5935811400413513,
      "distill": 1.0779372155666351,
      "teacher_acc": 0.7177843749523163
    },
    {
      "step": 5161,
      "lr": 3.9366892016277096e-05,
      "elapsed_s": 3836.22931432724,
      "loss": 3.160399615764618,
      "hidden": 0.20244242250919342,
      "layer": 0.0950684193521738,
      "mlm": 1.5560625493526459,
      "distill": 1.1519179940223694,
      "teacher_acc": 0.7329987734556198
    },
    {
      "step": 5171,
      "lr": 3.932566065954553e-05,
      "elapsed_s": 3843.59894323349,
      "loss": 3.117943584918976,
      "hidden": 0.2046220451593399,
      "layer": 0.09402966313064098,
      "mlm": 1.530993402004242,
      "distill": 1.130691260099411,
      "teacher_acc": 0.7239188998937607
    },
    {
      "step": 5181,
      "lr": 3.928437120429502e-05,
      "elapsed_s": 3850.981483221054,
      "loss": 3.240496516227722,
      "hidden": 0.20696918666362762,
      "layer": 0.09686686843633652,
      "mlm": 1.5881038010120392,
      "distill": 1.1900207996368408,
      "teacher_acc": 0.7208735197782516
    },
    {
      "step": 5191,
      "lr": 3.9243023817977264e-05,
      "elapsed_s": 3858.3763728141785,
      "loss": 3.315925180912018,
      "hidden": 0.20436597988009453,
      "layer": 0.09535984136164188,
      "mlm": 1.6764420568943024,
      "distill": 1.1830712854862213,
      "teacher_acc": 0.7128686457872391
    },
    {
      "step": 5201,
      "lr": 3.920161866827889e-05,
      "elapsed_s": 3865.7784128189087,
      "loss": 3.1283793449401855,
      "hidden": 0.19815217703580856,
      "layer": 0.09389823116362095,
      "mlm": 1.5900116264820099,
      "distill": 1.0951142609119415,
      "teacher_acc": 0.7098075151443481
    },
    {
      "step": 5211,
      "lr": 3.916015592312082e-05,
      "elapsed_s": 3873.1706261634827,
      "loss": 3.154094636440277,
      "hidden": 0.2045869454741478,
      "layer": 0.09474603831768036,
      "mlm": 1.5561692118644714,
      "distill": 1.14137864112854,
      "teacher_acc": 0.7365899980068207
    },
    {
      "step": 5221,
      "lr": 3.911863575065753e-05,
      "elapsed_s": 3880.5756254196167,
      "loss": 3.2967916131019592,
      "hidden": 0.20082442462444305,
      "layer": 0.09492165967822075,
      "mlm": 1.6190398633480072,
      "distill": 1.228642076253891,
      "teacher_acc": 0.7256738543510437
    },
    {
      "step": 5231,
      "lr": 3.9077058319276395e-05,
      "elapsed_s": 3887.9752378463745,
      "loss": 3.1839446425437927,
      "hidden": 0.20127076283097267,
      "layer": 0.09562620706856251,
      "mlm": 1.5423905849456787,
      "distill": 1.1911994516849518,
      "teacher_acc": 0.7253284454345703
    },
    {
      "step": 5241,
      "lr": 3.9035423797597027e-05,
      "elapsed_s": 3895.364002227783,
      "loss": 3.158566176891327,
      "hidden": 0.20146454125642776,
      "layer": 0.09579511545598507,
      "mlm": 1.5601206719875336,
      "distill": 1.1476188004016876,
      "teacher_acc": 0.7257874757051468
    },
    {
      "step": 5251,
      "lr": 3.8993732354470556e-05,
      "elapsed_s": 3902.749901533127,
      "loss": 3.3215824961662292,
      "hidden": 0.20382334291934967,
      "layer": 0.09563855081796646,
      "mlm": 1.6733167171478271,
      "distill": 1.1927998065948486,
      "teacher_acc": 0.7127123326063156
    },
    {
      "step": 5261,
      "lr": 3.895198415897896e-05,
      "elapsed_s": 3910.1876792907715,
      "loss": 3.133697032928467,
      "hidden": 0.20069924369454384,
      "layer": 0.09434601850807667,
      "mlm": 1.5828094780445099,
      "distill": 1.1023161113262177,
      "teacher_acc": 0.724223792552948
    },
    {
      "step": 5271,
      "lr": 3.8910179380434394e-05,
      "elapsed_s": 3917.5856449604034,
      "loss": 3.070586919784546,
      "hidden": 0.1927967108786106,
      "layer": 0.0928997453302145,
      "mlm": 1.5592449009418488,
      "distill": 1.0792986750602722,
      "teacher_acc": 0.7200212776660919
    },
    {
      "step": 5281,
      "lr": 3.886831818837847e-05,
      "elapsed_s": 3924.9761955738068,
      "loss": 3.25690758228302,
      "hidden": 0.20462371036410332,
      "layer": 0.09762348234653473,
      "mlm": 1.557576447725296,
      "distill": 1.2412719428539276,
      "teacher_acc": 0.7409348636865616
    },
    {
      "step": 5291,
      "lr": 3.88264007525816e-05,
      "elapsed_s": 3932.4134068489075,
      "loss": 3.0648136138916016,
      "hidden": 0.18844493478536606,
      "layer": 0.09216207824647427,
      "mlm": 1.6049093902111053,
      "distill": 1.036933422088623,
      "teacher_acc": 0.7005497962236404
    },
    {
      "step": 5301,
      "lr": 3.8784427243042296e-05,
      "elapsed_s": 3939.8023250102997,
      "loss": 3.1235196590423584,
      "hidden": 0.19936472922563553,
      "layer": 0.09414207376539707,
      "mlm": 1.5542917251586914,
      "distill": 1.1234274804592133,
      "teacher_acc": 0.7233496010303497
    },
    {
      "step": 5311,
      "lr": 3.8742397829986465e-05,
      "elapsed_s": 3947.2039272785187,
      "loss": 3.236342191696167,
      "hidden": 0.20166560262441635,
      "layer": 0.09504166059195995,
      "mlm": 1.5434348583221436,
      "distill": 1.2420552372932434,
      "teacher_acc": 0.7431695461273193
    },
    {
      "step": 5321,
      "lr": 3.870031268386676e-05,
      "elapsed_s": 3954.6426210403442,
      "loss": 3.2960351705551147,
      "hidden": 0.20759271085262299,
      "layer": 0.09561016969382763,
      "mlm": 1.6015039086341858,
      "distill": 1.2315407693386078,
      "teacher_acc": 0.7288808971643448
    },
    {
      "step": 5331,
      "lr": 3.865817197536182e-05,
      "elapsed_s": 3962.060968875885,
      "loss": 3.268190860748291,
      "hidden": 0.20999006181955338,
      "layer": 0.09752126224339008,
      "mlm": 1.5789802074432373,
      "distill": 1.220469892024994,
      "teacher_acc": 0.7243476957082748
    },
    {
      "step": 5341,
      "lr": 3.861597587537568e-05,
      "elapsed_s": 3969.493040084839,
      "loss": 3.175324857234955,
      "hidden": 0.2002992145717144,
      "layer": 0.09486480429768562,
      "mlm": 1.624749481678009,
      "distill": 1.102544516324997,
      "teacher_acc": 0.7055502384901047
    },
    {
      "step": 5351,
      "lr": 3.857372455503697e-05,
      "elapsed_s": 3976.9188375473022,
      "loss": 3.1311481595039368,
      "hidden": 0.1972450613975525,
      "layer": 0.09321189112961292,
      "mlm": 1.6187196969985962,
      "distill": 1.0713323652744293,
      "teacher_acc": 0.7194712162017822
    },
    {
      "step": 5361,
      "lr": 3.853141818569829e-05,
      "elapsed_s": 3984.32852101326,
      "loss": 2.961795687675476,
      "hidden": 0.2043016441166401,
      "layer": 0.09525318257510662,
      "mlm": 1.395276963710785,
      "distill": 1.110288828611374,
      "teacher_acc": 0.7504487037658691
    },
    {
      "step": 5371,
      "lr": 3.8489056938935483e-05,
      "elapsed_s": 3991.7455537319183,
      "loss": 3.123678207397461,
      "hidden": 0.19621623307466507,
      "layer": 0.09361790493130684,
      "mlm": 1.6045169830322266,
      "distill": 1.079919695854187,
      "teacher_acc": 0.7027239501476288
    },
    {
      "step": 5381,
      "lr": 3.8446640986546966e-05,
      "elapsed_s": 3999.1811094284058,
      "loss": 3.175505220890045,
      "hidden": 0.19534580782055855,
      "layer": 0.0941611509770155,
      "mlm": 1.599379062652588,
      "distill": 1.1383539140224457,
      "teacher_acc": 0.7210898548364639
    },
    {
      "step": 5391,
      "lr": 3.840417050055301e-05,
      "elapsed_s": 4006.5692915916443,
      "loss": 3.003191828727722,
      "hidden": 0.1943880058825016,
      "layer": 0.09284947626292706,
      "mlm": 1.5219251811504364,
      "distill": 1.0460660606622696,
      "teacher_acc": 0.7238606214523315
    },
    {
      "step": 5401,
      "lr": 3.8361645653195026e-05,
      "elapsed_s": 4013.9449203014374,
      "loss": 3.155056059360504,
      "hidden": 0.1977628991007805,
      "layer": 0.09409528970718384,
      "mlm": 1.6211577355861664,
      "distill": 1.0913249254226685,
      "teacher_acc": 0.7144501209259033
    },
    {
      "step": 5411,
      "lr": 3.831906661693494e-05,
      "elapsed_s": 4021.356166124344,
      "loss": 3.2660038471221924,
      "hidden": 0.20346135646104813,
      "layer": 0.09643821977078915,
      "mlm": 1.6242025792598724,
      "distill": 1.1866594552993774,
      "teacher_acc": 0.7224602550268173
    },
    {
      "step": 5421,
      "lr": 3.8276433564454416e-05,
      "elapsed_s": 4028.758138656616,
      "loss": 3.194348931312561,
      "hidden": 0.20030442252755165,
      "layer": 0.0949259102344513,
      "mlm": 1.5808246433734894,
      "distill": 1.1654525101184845,
      "teacher_acc": 0.7303000837564468
    },
    {
      "step": 5431,
      "lr": 3.823374666865418e-05,
      "elapsed_s": 4036.1603627204895,
      "loss": 3.085331916809082,
      "hidden": 0.19843630865216255,
      "layer": 0.09373984299600124,
      "mlm": 1.5342360138893127,
      "distill": 1.1073534190654755,
      "teacher_acc": 0.7294028401374817
    },
    {
      "step": 5441,
      "lr": 3.819100610265332e-05,
      "elapsed_s": 4043.6091282367706,
      "loss": 3.182236135005951,
      "hidden": 0.2007373422384262,
      "layer": 0.09559807926416397,
      "mlm": 1.5722976922988892,
      "distill": 1.1606647670269012,
      "teacher_acc": 0.7159816771745682
    },
    {
      "step": 5451,
      "lr": 3.81482120397886e-05,
      "elapsed_s": 4051.0435740947723,
      "loss": 3.2362318634986877,
      "hidden": 0.2044500820338726,
      "layer": 0.09659877978265285,
      "mlm": 1.5844030678272247,
      "distill": 1.1946293115615845,
      "teacher_acc": 0.7308434247970581
    },
    {
      "step": 5461,
      "lr": 3.8105364653613744e-05,
      "elapsed_s": 4058.465926885605,
      "loss": 3.2365540266036987,
      "hidden": 0.20084622502326965,
      "layer": 0.09513790532946587,
      "mlm": 1.6189433336257935,
      "distill": 1.1683493554592133,
      "teacher_acc": 0.7162838578224182
    },
    {
      "step": 5471,
      "lr": 3.8062464117898724e-05,
      "elapsed_s": 4065.867518901825,
      "loss": 3.2040562629699707,
      "hidden": 0.19839361310005188,
      "layer": 0.09430081397294998,
      "mlm": 1.6644619703292847,
      "distill": 1.0956567823886871,
      "teacher_acc": 0.6927765756845474
    },
    {
      "step": 5481,
      "lr": 3.8019510606629064e-05,
      "elapsed_s": 4073.2898931503296,
      "loss": 3.2046201825141907,
      "hidden": 0.19593093171715736,
      "layer": 0.0933859571814537,
      "mlm": 1.6396242082118988,
      "distill": 1.1264412105083466,
      "teacher_acc": 0.713366910815239
    },
    {
      "step": 5491,
      "lr": 3.7976504294005125e-05,
      "elapsed_s": 4080.7035796642303,
      "loss": 3.1884191632270813,
      "hidden": 0.20163608342409134,
      "layer": 0.09460514783859253,
      "mlm": 1.5915963053703308,
      "distill": 1.146247997879982,
      "teacher_acc": 0.7161682844161987
    },
    {
      "step": 5501,
      "lr": 3.793344535444142e-05,
      "elapsed_s": 4088.109368801117,
      "loss": 3.290668547153473,
      "hidden": 0.19974730536341667,
      "layer": 0.09316955134272575,
      "mlm": 1.7141708135604858,
      "distill": 1.1304183304309845,
      "teacher_acc": 0.6968995779752731
    },
    {
      "step": 5511,
      "lr": 3.789033396256589e-05,
      "elapsed_s": 4095.519630908966,
      "loss": 3.137165367603302,
      "hidden": 0.1943967528641224,
      "layer": 0.09414484538137913,
      "mlm": 1.5985552668571472,
      "distill": 1.1027441322803497,
      "teacher_acc": 0.7089481949806213
    },
    {
      "step": 5521,
      "lr": 3.784717029321922e-05,
      "elapsed_s": 4102.946670293808,
      "loss": 3.0625978112220764,
      "hidden": 0.19923468306660652,
      "layer": 0.0952229555696249,
      "mlm": 1.5044777989387512,
      "distill": 1.1120392382144928,
      "teacher_acc": 0.7373954951763153
    },
    {
      "step": 5531,
      "lr": 3.7803954521454055e-05,
      "elapsed_s": 4110.364837408066,
      "loss": 3.133970618247986,
      "hidden": 0.20013427734375,
      "layer": 0.09483890421688557,
      "mlm": 1.5003079175949097,
      "distill": 1.1859746277332306,
      "teacher_acc": 0.7449219673871994
    },
    {
      "step": 5541,
      "lr": 3.77606868225344e-05,
      "elapsed_s": 4117.779819011688,
      "loss": 3.2288476824760437,
      "hidden": 0.20172005891799927,
      "layer": 0.09434804134070873,
      "mlm": 1.6403281092643738,
      "distill": 1.1379053890705109,
      "teacher_acc": 0.7073512226343155
    },
    {
      "step": 5551,
      "lr": 3.7717367371934805e-05,
      "elapsed_s": 4125.198168992996,
      "loss": 3.0198581218719482,
      "hidden": 0.19426102191209793,
      "layer": 0.09181394055485725,
      "mlm": 1.5087133944034576,
      "distill": 1.0767156481742859,
      "teacher_acc": 0.7334019988775253
    },
    {
      "step": 5561,
      "lr": 3.7673996345339765e-05,
      "elapsed_s": 4132.600310087204,
      "loss": 3.172568619251251,
      "hidden": 0.20363150537014008,
      "layer": 0.09725976921617985,
      "mlm": 1.4820390045642853,
      "distill": 1.2346367537975311,
      "teacher_acc": 0.7472864389419556
    },
    {
      "step": 5571,
      "lr": 3.763057391864287e-05,
      "elapsed_s": 4140.039600133896,
      "loss": 3.1457441449165344,
      "hidden": 0.19964130595326424,
      "layer": 0.09341919049620628,
      "mlm": 1.5451968610286713,
      "distill": 1.1545551121234894,
      "teacher_acc": 0.7254993468523026
    },
    {
      "step": 5581,
      "lr": 3.7587100267946226e-05,
      "elapsed_s": 4147.4629101753235,
      "loss": 3.1725448966026306,
      "hidden": 0.19636895135045052,
      "layer": 0.0932153444737196,
      "mlm": 1.6197852492332458,
      "distill": 1.1134141683578491,
      "teacher_acc": 0.7151211947202682
    },
    {
      "step": 5591,
      "lr": 3.754357556955966e-05,
      "elapsed_s": 4154.873417139053,
      "loss": 3.170562207698822,
      "hidden": 0.1973896026611328,
      "layer": 0.09508186392486095,
      "mlm": 1.574512094259262,
      "distill": 1.1537300646305084,
      "teacher_acc": 0.723589614033699
    },
    {
      "step": 5601,
      "lr": 3.7500000000000003e-05,
      "elapsed_s": 4162.27840924263,
      "loss": 3.2920291423797607,
      "hidden": 0.20497823506593704,
      "layer": 0.09670058265328407,
      "mlm": 1.6148390173912048,
      "distill": 1.2188833355903625,
      "teacher_acc": 0.720633253455162
    },
    {
      "step": 5611,
      "lr": 3.745637373599043e-05,
      "elapsed_s": 4169.682890176773,
      "loss": 3.0663285851478577,
      "hidden": 0.20180582255125046,
      "layer": 0.09377403371036053,
      "mlm": 1.486604005098343,
      "distill": 1.1292259991168976,
      "teacher_acc": 0.7422326356172562
    },
    {
      "step": 5621,
      "lr": 3.74126969544597e-05,
      "elapsed_s": 4177.069175720215,
      "loss": 3.1064653396606445,
      "hidden": 0.19697028398513794,
      "layer": 0.09409929066896439,
      "mlm": 1.5173876285552979,
      "distill": 1.1480874717235565,
      "teacher_acc": 0.7443343251943588
    },
    {
      "step": 5631,
      "lr": 3.736896983254144e-05,
      "elapsed_s": 4184.449201583862,
      "loss": 3.1099019646644592,
      "hidden": 0.19263843074440956,
      "layer": 0.09185251407325268,
      "mlm": 1.605536550283432,
      "distill": 1.0731623768806458,
      "teacher_acc": 0.7089554518461227
    },
    {
      "step": 5641,
      "lr": 3.732519254757344e-05,
      "elapsed_s": 4191.847795248032,
      "loss": 3.151432514190674,
      "hidden": 0.19703428074717522,
      "layer": 0.09372341074049473,
      "mlm": 1.5938798785209656,
      "distill": 1.1166223734617233,
      "teacher_acc": 0.7129863351583481
    },
    {
      "step": 5651,
      "lr": 3.728136527709694e-05,
      "elapsed_s": 4199.248284339905,
      "loss": 3.1422083973884583,
      "hidden": 0.19626714289188385,
      "layer": 0.09301349148154259,
      "mlm": 1.5916087925434113,
      "distill": 1.111558586359024,
      "teacher_acc": 0.7172615081071854
    },
    {
      "step": 5661,
      "lr": 3.7237488198855876e-05,
      "elapsed_s": 4206.671406984329,
      "loss": 3.204590380191803,
      "hidden": 0.2027047574520111,
      "layer": 0.09622904472053051,
      "mlm": 1.5687493085861206,
      "distill": 1.1823169738054276,
      "teacher_acc": 0.7289220541715622
    },
    {
      "step": 5671,
      "lr": 3.719356149079621e-05,
      "elapsed_s": 4214.110413312912,
      "loss": 3.0182340145111084,
      "hidden": 0.19068153202533722,
      "layer": 0.09195774979889393,
      "mlm": 1.5081001818180084,
      "distill": 1.0827918648719788,
      "teacher_acc": 0.7312351614236832
    },
    {
      "step": 5681,
      "lr": 3.714958533106515e-05,
      "elapsed_s": 4221.516685962677,
      "loss": 3.059345841407776,
      "hidden": 0.19623732566833496,
      "layer": 0.09316440485417843,
      "mlm": 1.5153020322322845,
      "distill": 1.1049869656562805,
      "teacher_acc": 0.7332832366228104
    },
    {
      "step": 5691,
      "lr": 3.7105559898010494e-05,
      "elapsed_s": 4228.908064842224,
      "loss": 3.1445667147636414,
      "hidden": 0.19566473364830017,
      "layer": 0.09415927529335022,
      "mlm": 1.6187961399555206,
      "distill": 1.0873614251613617,
      "teacher_acc": 0.7032727003097534
    },
    {
      "step": 5701,
      "lr": 3.7061485370179835e-05,
      "elapsed_s": 4236.31845664978,
      "loss": 3.2046861052513123,
      "hidden": 0.19997898116707802,
      "layer": 0.0945882722735405,
      "mlm": 1.6236422955989838,
      "distill": 1.1337918043136597,
      "teacher_acc": 0.7056026309728622
    },
    {
      "step": 5711,
      "lr": 3.7017361926319896e-05,
      "elapsed_s": 4243.7177884578705,
      "loss": 3.1038458943367004,
      "hidden": 0.19729091599583626,
      "layer": 0.09275224432349205,
      "mlm": 1.586482435464859,
      "distill": 1.0764055252075195,
      "teacher_acc": 0.7129049301147461
    },
    {
      "step": 5721,
      "lr": 3.697318974537577e-05,
      "elapsed_s": 4251.103879213333,
      "loss": 3.051403284072876,
      "hidden": 0.19367704913020134,
      "layer": 0.09298089891672134,
      "mlm": 1.5436148047447205,
      "distill": 1.0739439129829407,
      "teacher_acc": 0.7275170981884003
    },
    {
      "step": 5731,
      "lr": 3.692896900649021e-05,
      "elapsed_s": 4258.504755020142,
      "loss": 3.067038655281067,
      "hidden": 0.19476135447621346,
      "layer": 0.0927730593830347,
      "mlm": 1.5237123668193817,
      "distill": 1.1074170470237732,
      "teacher_acc": 0.7217403054237366
    },
    {
      "step": 5741,
      "lr": 3.688469988900291e-05,
      "elapsed_s": 4265.913398742676,
      "loss": 3.1705139875411987,
      "hidden": 0.1984352320432663,
      "layer": 0.09342442825436592,
      "mlm": 1.6045320630073547,
      "distill": 1.1223991811275482,
      "teacher_acc": 0.7147362381219864
    },
    {
      "step": 5751,
      "lr": 3.684038257244973e-05,
      "elapsed_s": 4273.349415063858,
      "loss": 3.198856294155121,
      "hidden": 0.19722573086619377,
      "layer": 0.09326377883553505,
      "mlm": 1.6367175579071045,
      "distill": 1.1210553050041199,
      "teacher_acc": 0.7134540975093842
    },
    {
      "step": 5761,
      "lr": 3.679601723656205e-05,
      "elapsed_s": 4280.759192228317,
      "loss": 3.169144868850708,
      "hidden": 0.20101090148091316,
      "layer": 0.09429389424622059,
      "mlm": 1.563440889120102,
      "distill": 1.1565352976322174,
      "teacher_acc": 0.7204267382621765
    },
    {
      "step": 5771,
      "lr": 3.675160406126598e-05,
      "elapsed_s": 4288.193029165268,
      "loss": 2.99551260471344,
      "hidden": 0.1948242001235485,
      "layer": 0.09313718788325787,
      "mlm": 1.48704132437706,
      "distill": 1.0722542703151703,
      "teacher_acc": 0.7360995709896088
    },
    {
      "step": 5781,
      "lr": 3.670714322668162e-05,
      "elapsed_s": 4295.629599094391,
      "loss": 3.1230621337890625,
      "hidden": 0.1968647502362728,
      "layer": 0.09324252419173717,
      "mlm": 1.6089228093624115,
      "distill": 1.073788583278656,
      "teacher_acc": 0.714422807097435
    },
    {
      "step": 5791,
      "lr": 3.666263491312238e-05,
      "elapsed_s": 4303.059733390808,
      "loss": 3.0092450380325317,
      "hidden": 0.19525746628642082,
      "layer": 0.09211529977619648,
      "mlm": 1.4981193840503693,
      "distill": 1.0745531022548676,
      "teacher_acc": 0.7241339385509491
    },
    {
      "step": 5801,
      "lr": 3.6618079301094216e-05,
      "elapsed_s": 4310.493505001068,
      "loss": 3.0423428416252136,
      "hidden": 0.19879037514328957,
      "layer": 0.0931391753256321,
      "mlm": 1.479296863079071,
      "distill": 1.118895635008812,
      "teacher_acc": 0.7410182952880859
    },
    {
      "step": 5811,
      "lr": 3.657347657129491e-05,
      "elapsed_s": 4317.898047208786,
      "loss": 3.1755396127700806,
      "hidden": 0.19853822141885757,
      "layer": 0.0937601588666439,
      "mlm": 1.583716720342636,
      "distill": 1.1478663980960846,
      "teacher_acc": 0.7128798514604568
    },
    {
      "step": 5821,
      "lr": 3.652882690461334e-05,
      "elapsed_s": 4325.2860243320465,
      "loss": 3.2515771985054016,
      "hidden": 0.19827789068222046,
      "layer": 0.09446846134960651,
      "mlm": 1.602838009595871,
      "distill": 1.2049491703510284,
      "teacher_acc": 0.7194454222917557
    },
    {
      "step": 5831,
      "lr": 3.6484130482128715e-05,
      "elapsed_s": 4332.703218460083,
      "loss": 3.086265802383423,
      "hidden": 0.1916314736008644,
      "layer": 0.09231014735996723,
      "mlm": 1.599512666463852,
      "distill": 1.0573351681232452,
      "teacher_acc": 0.7085776925086975
    },
    {
      "step": 5841,
      "lr": 3.643938748510989e-05,
      "elapsed_s": 4340.10419178009,
      "loss": 3.0327399373054504,
      "hidden": 0.1957053691148758,
      "layer": 0.09257950261235237,
      "mlm": 1.539845734834671,
      "distill": 1.0551936775445938,
      "teacher_acc": 0.7189159691333771
    },
    {
      "step": 5851,
      "lr": 3.639459809501458e-05,
      "elapsed_s": 4347.500159978867,
      "loss": 3.088723659515381,
      "hidden": 0.19210728257894516,
      "layer": 0.0919422097504139,
      "mlm": 1.5582979321479797,
      "distill": 1.1002400517463684,
      "teacher_acc": 0.7227726727724075
    },
    {
      "step": 5861,
      "lr": 3.634976249348867e-05,
      "elapsed_s": 4354.860340356827,
      "loss": 3.0584779381752014,
      "hidden": 0.1946190930902958,
      "layer": 0.09212624467909336,
      "mlm": 1.5802371203899384,
      "distill": 1.0429396033287048,
      "teacher_acc": 0.7077350914478302
    },
    {
      "step": 5871,
      "lr": 3.630488086236547e-05,
      "elapsed_s": 4362.229564666748,
      "loss": 3.1185101866722107,
      "hidden": 0.2037334106862545,
      "layer": 0.09478634223341942,
      "mlm": 1.4842193126678467,
      "distill": 1.1794309318065643,
      "teacher_acc": 0.7386865317821503
    },
    {
      "step": 5881,
      "lr": 3.625995338366492e-05,
      "elapsed_s": 4369.605774641037,
      "loss": 3.1529953479766846,
      "hidden": 0.19466375559568405,
      "layer": 0.09312479197978973,
      "mlm": 1.6404542326927185,
      "distill": 1.0766512602567673,
      "teacher_acc": 0.6927720010280609
    },
    {
      "step": 5891,
      "lr": 3.621498023959294e-05,
      "elapsed_s": 4376.983222961426,
      "loss": 3.1072345972061157,
      "hidden": 0.19316312670707703,
      "layer": 0.09175510704517365,
      "mlm": 1.5583978593349457,
      "distill": 1.1166329085826874,
      "teacher_acc": 0.7226730734109879
    },
    {
      "step": 5901,
      "lr": 3.6169961612540645e-05,
      "elapsed_s": 4384.361966371536,
      "loss": 3.1410861015319824,
      "hidden": 0.19642717018723488,
      "layer": 0.0932257566601038,
      "mlm": 1.6337474584579468,
      "distill": 1.0678713917732239,
      "teacher_acc": 0.7094090729951859
    },
    {
      "step": 5911,
      "lr": 3.6124897685083584e-05,
      "elapsed_s": 4391.746059417725,
      "loss": 3.122596859931946,
      "hidden": 0.19022661074995995,
      "layer": 0.09268088825047016,
      "mlm": 1.6028250753879547,
      "distill": 1.0929780900478363,
      "teacher_acc": 0.7101231068372726
    },
    {
      "step": 5921,
      "lr": 3.6079788639981036e-05,
      "elapsed_s": 4399.142077207565,
      "loss": 3.0488314628601074,
      "hidden": 0.1918628141283989,
      "layer": 0.09250980243086815,
      "mlm": 1.5653907358646393,
      "distill": 1.053460106253624,
      "teacher_acc": 0.7140750288963318
    },
    {
      "step": 5931,
      "lr": 3.603463466017526e-05,
      "elapsed_s": 4406.547158718109,
      "loss": 3.2016438245773315,
      "hidden": 0.2022787220776081,
      "layer": 0.09421056509017944,
      "mlm": 1.6026907861232758,
      "distill": 1.1472904086112976,
      "teacher_acc": 0.7194232791662216
    },
    {
      "step": 5941,
      "lr": 3.598943592879076e-05,
      "elapsed_s": 4413.961395740509,
      "loss": 3.0697553753852844,
      "hidden": 0.19046977534890175,
      "layer": 0.09219641424715519,
      "mlm": 1.5915293097496033,
      "distill": 1.0511883199214935,
      "teacher_acc": 0.7116304337978363
    },
    {
      "step": 5951,
      "lr": 3.594419262913351e-05,
      "elapsed_s": 4421.36256814003,
      "loss": 3.200571119785309,
      "hidden": 0.19930657744407654,
      "layer": 0.09469421580433846,
      "mlm": 1.5993623733520508,
      "distill": 1.155248373746872,
      "teacher_acc": 0.7190072238445282
    },
    {
      "step": 5961,
      "lr": 3.5898904944690256e-05,
      "elapsed_s": 4428.778516054153,
      "loss": 3.1300723552703857,
      "hidden": 0.1962214782834053,
      "layer": 0.09300321899354458,
      "mlm": 1.535264402627945,
      "distill": 1.1558633297681808,
      "teacher_acc": 0.728346198797226
    },
    {
      "step": 5971,
      "lr": 3.585357305912773e-05,
      "elapsed_s": 4436.189889907837,
      "loss": 3.0895994305610657,
      "hidden": 0.1965099312365055,
      "layer": 0.09366054646670818,
      "mlm": 1.5356993675231934,
      "distill": 1.1140500605106354,
      "teacher_acc": 0.7267555892467499
    },
    {
      "step": 5981,
      "lr": 3.5808197156291937e-05,
      "elapsed_s": 4443.58362698555,
      "loss": 3.1897528171539307,
      "hidden": 0.19996138289570808,
      "layer": 0.09377051703631878,
      "mlm": 1.614856481552124,
      "distill": 1.128088265657425,
      "teacher_acc": 0.7175811231136322
    },
    {
      "step": 5991,
      "lr": 3.576277742020738e-05,
      "elapsed_s": 4450.93958067894,
      "loss": 3.1116180419921875,
      "hidden": 0.18896225839853287,
      "layer": 0.09107259847223759,
      "mlm": 1.5880478024482727,
      "distill": 1.1001093983650208,
      "teacher_acc": 0.7248478829860687
    },
    {
      "step": 6001,
      "lr": 3.5717314035076355e-05,
      "elapsed_s": 4460.279916763306,
      "loss": 3.104872941970825,
      "hidden": 0.19399436563253403,
      "layer": 0.09274009987711906,
      "mlm": 1.5995230972766876,
      "distill": 1.0709910988807678,
      "teacher_acc": 0.707163542509079
    },
    {
      "step": 6011,
      "lr": 3.567180718527816e-05,
      "elapsed_s": 4467.658266067505,
      "loss": 3.192289650440216,
      "hidden": 0.19655533879995346,
      "layer": 0.09295963495969772,
      "mlm": 1.6005885303020477,
      "distill": 1.152110606431961,
      "teacher_acc": 0.7297551035881042
    },
    {
      "step": 6021,
      "lr": 3.5626257055368386e-05,
      "elapsed_s": 4475.001597881317,
      "loss": 3.0388848781585693,
      "hidden": 0.19529186934232712,
      "layer": 0.09242597594857216,
      "mlm": 1.559584230184555,
      "distill": 1.0425038933753967,
      "teacher_acc": 0.7169177681207657
    },
    {
      "step": 6031,
      "lr": 3.558066383007811e-05,
      "elapsed_s": 4482.408060789108,
      "loss": 3.190236985683441,
      "hidden": 0.19664927944540977,
      "layer": 0.09323549456894398,
      "mlm": 1.6449633240699768,
      "distill": 1.1053574085235596,
      "teacher_acc": 0.7008141577243805
    },
    {
      "step": 6041,
      "lr": 3.553502769431323e-05,
      "elapsed_s": 4489.808842658997,
      "loss": 3.0385810136795044,
      "hidden": 0.19561300799250603,
      "layer": 0.09295208565890789,
      "mlm": 1.4677660167217255,
      "distill": 1.133112907409668,
      "teacher_acc": 0.7505103051662445
    },
    {
      "step": 6051,
      "lr": 3.5489348833153645e-05,
      "elapsed_s": 4497.222655534744,
      "loss": 2.949130356311798,
      "hidden": 0.18863296136260033,
      "layer": 0.09131281450390816,
      "mlm": 1.5205333232879639,
      "distill": 1.0056747049093246,
      "teacher_acc": 0.7218647748231888
    },
    {
      "step": 6061,
      "lr": 3.5443627431852526e-05,
      "elapsed_s": 4504.646123886108,
      "loss": 3.3349632620811462,
      "hidden": 0.2100144326686859,
      "layer": 0.096647834405303,
      "mlm": 1.6618981659412384,
      "distill": 1.2047122567892075,
      "teacher_acc": 0.7154448330402374
    },
    {
      "step": 6071,
      "lr": 3.539786367583559e-05,
      "elapsed_s": 4512.040123939514,
      "loss": 3.091992497444153,
      "hidden": 0.19086800888180733,
      "layer": 0.09352634474635124,
      "mlm": 1.569416880607605,
      "distill": 1.0940764546394348,
      "teacher_acc": 0.713665172457695
    },
    {
      "step": 6081,
      "lr": 3.53520577507003e-05,
      "elapsed_s": 4519.456461906433,
      "loss": 3.1018927097320557,
      "hidden": 0.19065378978848457,
      "layer": 0.09131838567554951,
      "mlm": 1.612713247537613,
      "distill": 1.062212586402893,
      "teacher_acc": 0.7119351029396057
    },
    {
      "step": 6091,
      "lr": 3.530620984221517e-05,
      "elapsed_s": 4526.869574069977,
      "loss": 3.012139081954956,
      "hidden": 0.19279375672340393,
      "layer": 0.09181579574942589,
      "mlm": 1.5358278453350067,
      "distill": 1.044815868139267,
      "teacher_acc": 0.7169939875602722
    },
    {
      "step": 6101,
      "lr": 3.526032013631893e-05,
      "elapsed_s": 4534.259119749069,
      "loss": 3.0672601461410522,
      "hidden": 0.19286799430847168,
      "layer": 0.09271782264113426,
      "mlm": 1.4937605261802673,
      "distill": 1.1414046883583069,
      "teacher_acc": 0.738550990819931
    },
    {
      "step": 6111,
      "lr": 3.5214388819119864e-05,
      "elapsed_s": 4541.676528930664,
      "loss": 3.055286705493927,
      "hidden": 0.1949954628944397,
      "layer": 0.0916488841176033,
      "mlm": 1.5192622542381287,
      "distill": 1.1002090722322464,
      "teacher_acc": 0.7315688729286194
    },
    {
      "step": 6121,
      "lr": 3.516841607689501e-05,
      "elapsed_s": 4549.03208231926,
      "loss": 3.195590913295746,
      "hidden": 0.19239870086312294,
      "layer": 0.09298037923872471,
      "mlm": 1.6326356530189514,
      "distill": 1.131667673587799,
      "teacher_acc": 0.7100181132555008
    },
    {
      "step": 6131,
      "lr": 3.512240209608938e-05,
      "elapsed_s": 4556.368154525757,
      "loss": 3.0735949873924255,
      "hidden": 0.1926220767199993,
      "layer": 0.09290842711925507,
      "mlm": 1.5498071014881134,
      "distill": 1.0920895338058472,
      "teacher_acc": 0.7250847518444061
    },
    {
      "step": 6141,
      "lr": 3.507634706331528e-05,
      "elapsed_s": 4563.721066951752,
      "loss": 3.0311434864997864,
      "hidden": 0.1887418068945408,
      "layer": 0.09088263474404812,
      "mlm": 1.584464967250824,
      "distill": 1.023753508925438,
      "teacher_acc": 0.7064801007509232
    },
    {
      "step": 6151,
      "lr": 3.5030251165351445e-05,
      "elapsed_s": 4571.082675695419,
      "loss": 3.0998308658599854,
      "hidden": 0.19653206691145897,
      "layer": 0.09295044653117657,
      "mlm": 1.5395187139511108,
      "distill": 1.120772808790207,
      "teacher_acc": 0.7304093241691589
    },
    {
      "step": 6161,
      "lr": 3.498411458914238e-05,
      "elapsed_s": 4578.449160814285,
      "loss": 3.131224513053894,
      "hidden": 0.20197837427258492,
      "layer": 0.0953552145510912,
      "mlm": 1.5389502346515656,
      "distill": 1.1406399607658386,
      "teacher_acc": 0.7261212170124054
    },
    {
      "step": 6171,
      "lr": 3.493793752179757e-05,
      "elapsed_s": 4585.777254581451,
      "loss": 3.2380929589271545,
      "hidden": 0.19713333621621132,
      "layer": 0.0939996987581253,
      "mlm": 1.6172708570957184,
      "distill": 1.179555594921112,
      "teacher_acc": 0.7169123888015747
    },
    {
      "step": 6181,
      "lr": 3.4891720150590665e-05,
      "elapsed_s": 4593.138922452927,
      "loss": 3.0120063424110413,
      "hidden": 0.19103293120861053,
      "layer": 0.09219476580619812,
      "mlm": 1.4685065150260925,
      "distill": 1.115336537361145,
      "teacher_acc": 0.7322067469358444
    },
    {
      "step": 6191,
      "lr": 3.484546266295883e-05,
      "elapsed_s": 4600.499802827835,
      "loss": 3.0194032192230225,
      "hidden": 0.18530699238181114,
      "layer": 0.09096686542034149,
      "mlm": 1.5847181379795074,
      "distill": 1.018587738275528,
      "teacher_acc": 0.7122793942689896
    },
    {
      "step": 6201,
      "lr": 3.479916524650188e-05,
      "elapsed_s": 4607.893656253815,
      "loss": 3.1598125100135803,
      "hidden": 0.1954488754272461,
      "layer": 0.0930603425949812,
      "mlm": 1.6145078539848328,
      "distill": 1.107876718044281,
      "teacher_acc": 0.7155182659626007
    },
    {
      "step": 6211,
      "lr": 3.4752828088981584e-05,
      "elapsed_s": 4615.286079645157,
      "loss": 2.980879783630371,
      "hidden": 0.19297680258750916,
      "layer": 0.09258772619068623,
      "mlm": 1.4769399464130402,
      "distill": 1.0716923475265503,
      "teacher_acc": 0.7333994060754776
    },
    {
      "step": 6221,
      "lr": 3.470645137832087e-05,
      "elapsed_s": 4622.7002012729645,
      "loss": 3.074561297893524,
      "hidden": 0.1925695240497589,
      "layer": 0.09185802191495895,
      "mlm": 1.4929607510566711,
      "distill": 1.150532603263855,
      "teacher_acc": 0.7423826605081558
    },
    {
      "step": 6231,
      "lr": 3.46600353026031e-05,
      "elapsed_s": 4630.108730554581,
      "loss": 3.13653302192688,
      "hidden": 0.19634933024644852,
      "layer": 0.09385943599045277,
      "mlm": 1.5030692517757416,
      "distill": 1.1938354074954987,
      "teacher_acc": 0.7433117628097534
    },
    {
      "step": 6241,
      "lr": 3.461358005007128e-05,
      "elapsed_s": 4637.474723100662,
      "loss": 3.1633260846138,
      "hidden": 0.1962810531258583,
      "layer": 0.09334804303944111,
      "mlm": 1.6518783271312714,
      "distill": 1.072211593389511,
      "teacher_acc": 0.6963322460651398
    },
    {
      "step": 6251,
      "lr": 3.456708580912725e-05,
      "elapsed_s": 4644.8981194496155,
      "loss": 3.2563687562942505,
      "hidden": 0.19847261533141136,
      "layer": 0.09264726936817169,
      "mlm": 1.6695808470249176,
      "distill": 1.143519103527069,
      "teacher_acc": 0.7036411464214325
    },
    {
      "step": 6261,
      "lr": 3.452055276833103e-05,
      "elapsed_s": 4652.290805101395,
      "loss": 3.1242244243621826,
      "hidden": 0.1947631500661373,
      "layer": 0.09387802891433239,
      "mlm": 1.612898826599121,
      "distill": 1.0748602598905563,
      "teacher_acc": 0.7127125412225723
    },
    {
      "step": 6271,
      "lr": 3.447398111639997e-05,
      "elapsed_s": 4659.686680793762,
      "loss": 3.093975841999054,
      "hidden": 0.19122236594557762,
      "layer": 0.09244493581354618,
      "mlm": 1.5185782313346863,
      "distill": 1.1467305123806,
      "teacher_acc": 0.7303605824708939
    },
    {
      "step": 6281,
      "lr": 3.442737104220801e-05,
      "elapsed_s": 4667.111065626144,
      "loss": 3.1411534547805786,
      "hidden": 0.19359271600842476,
      "layer": 0.09279880858957767,
      "mlm": 1.5820659697055817,
      "distill": 1.1255027055740356,
      "teacher_acc": 0.7189823538064957
    },
    {
      "step": 6291,
      "lr": 3.438072273478493e-05,
      "elapsed_s": 4674.509824991226,
      "loss": 2.973774790763855,
      "hidden": 0.18445955216884613,
      "layer": 0.09088463708758354,
      "mlm": 1.5108748078346252,
      "distill": 1.0485386103391647,
      "teacher_acc": 0.7243620008230209
    },
    {
      "step": 6301,
      "lr": 3.433403638331553e-05,
      "elapsed_s": 4681.923840522766,
      "loss": 3.1949153542518616,
      "hidden": 0.1940714418888092,
      "layer": 0.09177921526134014,
      "mlm": 1.6279211044311523,
      "distill": 1.1329616904258728,
      "teacher_acc": 0.7126029282808304
    },
    {
      "step": 6311,
      "lr": 3.428731217713893e-05,
      "elapsed_s": 4689.32967042923,
      "loss": 3.1435269713401794,
      "hidden": 0.1929805986583233,
      "layer": 0.0920780636370182,
      "mlm": 1.6183415651321411,
      "distill": 1.0931851863861084,
      "teacher_acc": 0.7095890939235687
    },
    {
      "step": 6321,
      "lr": 3.424055030574778e-05,
      "elapsed_s": 4696.747277021408,
      "loss": 2.9731759428977966,
      "hidden": 0.19011976197361946,
      "layer": 0.09108774363994598,
      "mlm": 1.506452888250351,
      "distill": 1.040939599275589,
      "teacher_acc": 0.7180880010128021
    },
    {
      "step": 6331,
      "lr": 3.419375095878744e-05,
      "elapsed_s": 4704.159867048264,
      "loss": 2.9945162534713745,
      "hidden": 0.18684181943535805,
      "layer": 0.09163513779640198,
      "mlm": 1.5197808742523193,
      "distill": 1.0552342236042023,
      "teacher_acc": 0.7267776876688004
    },
    {
      "step": 6341,
      "lr": 3.414691432605531e-05,
      "elapsed_s": 4711.619735956192,
      "loss": 3.089985728263855,
      "hidden": 0.19372475519776344,
      "layer": 0.09256158955395222,
      "mlm": 1.5048571527004242,
      "distill": 1.1513982713222504,
      "teacher_acc": 0.7379518151283264
    },
    {
      "step": 6351,
      "lr": 3.410004059749996e-05,
      "elapsed_s": 4719.052491188049,
      "loss": 3.0789005756378174,
      "hidden": 0.1967955306172371,
      "layer": 0.09230436943471432,
      "mlm": 1.5036981701850891,
      "distill": 1.135459154844284,
      "teacher_acc": 0.7352399528026581
    },
    {
      "step": 6361,
      "lr": 3.405312996322042e-05,
      "elapsed_s": 4726.476742744446,
      "loss": 3.0318517684936523,
      "hidden": 0.18728180974721909,
      "layer": 0.09061502665281296,
      "mlm": 1.5571210980415344,
      "distill": 1.054859533905983,
      "teacher_acc": 0.7130605429410934
    },
    {
      "step": 6371,
      "lr": 3.400618261346541e-05,
      "elapsed_s": 4733.878169059753,
      "loss": 3.032357692718506,
      "hidden": 0.18507330864667892,
      "layer": 0.09065103530883789,
      "mlm": 1.560399204492569,
      "distill": 1.0564863234758377,
      "teacher_acc": 0.7204049080610275
    },
    {
      "step": 6381,
      "lr": 3.39591987386325e-05,
      "elapsed_s": 4741.28510761261,
      "loss": 3.0905752182006836,
      "hidden": 0.19499319791793823,
      "layer": 0.09273175895214081,
      "mlm": 1.5539273619651794,
      "distill": 1.100295603275299,
      "teacher_acc": 0.7185826301574707
    },
    {
      "step": 6391,
      "lr": 3.391217852926748e-05,
      "elapsed_s": 4748.716134309769,
      "loss": 3.162065804004669,
      "hidden": 0.19218286499381065,
      "layer": 0.09223482571542263,
      "mlm": 1.6156118214130402,
      "distill": 1.1159708499908447,
      "teacher_acc": 0.7138348519802094
    },
    {
      "step": 6401,
      "lr": 3.386512217606339e-05,
      "elapsed_s": 4756.144778251648,
      "loss": 3.103502035140991,
      "hidden": 0.1960962563753128,
      "layer": 0.09236952103674412,
      "mlm": 1.532025545835495,
      "distill": 1.1330991983413696,
      "teacher_acc": 0.7234564572572708
    },
    {
      "step": 6411,
      "lr": 3.3818029869859936e-05,
      "elapsed_s": 4763.560351133347,
      "loss": 3.100357413291931,
      "hidden": 0.19265703111886978,
      "layer": 0.09179454669356346,
      "mlm": 1.6009341776371002,
      "distill": 1.068211942911148,
      "teacher_acc": 0.7109077572822571
    },
    {
      "step": 6421,
      "lr": 3.3770901801642595e-05,
      "elapsed_s": 4770.972047805786,
      "loss": 3.0820988416671753,
      "hidden": 0.19485889747738838,
      "layer": 0.09259810857474804,
      "mlm": 1.5118177235126495,
      "distill": 1.134264349937439,
      "teacher_acc": 0.7367132753133774
    },
    {
      "step": 6431,
      "lr": 3.3723738162541876e-05,
      "elapsed_s": 4778.382648944855,
      "loss": 3.0686978101730347,
      "hidden": 0.19695492461323738,
      "layer": 0.09331716783344746,
      "mlm": 1.4770510792732239,
      "distill": 1.1510784029960632,
      "teacher_acc": 0.7403425425291061
    },
    {
      "step": 6441,
      "lr": 3.367653914383257e-05,
      "elapsed_s": 4785.777971744537,
      "loss": 3.0721373558044434,
      "hidden": 0.19112543389201164,
      "layer": 0.09181712381541729,
      "mlm": 1.5336134135723114,
      "distill": 1.1103644967079163,
      "teacher_acc": 0.7308970391750336
    },
    {
      "step": 6451,
      "lr": 3.362930493693295e-05,
      "elapsed_s": 4793.159542798996,
      "loss": 2.9649736881256104,
      "hidden": 0.18709658831357956,
      "layer": 0.09151283837854862,
      "mlm": 1.4864593744277954,
      "distill": 1.058564692735672,
      "teacher_acc": 0.7190053462982178
    },
    {
      "step": 6461,
      "lr": 3.358203573340396e-05,
      "elapsed_s": 4800.549548864365,
      "loss": 3.054726719856262,
      "hidden": 0.1893276758491993,
      "layer": 0.0914696604013443,
      "mlm": 1.572209745645523,
      "distill": 1.0581267476081848,
      "teacher_acc": 0.7191956490278244
    },
    {
      "step": 6471,
      "lr": 3.353473172494852e-05,
      "elapsed_s": 4807.918670415878,
      "loss": 3.14129638671875,
      "hidden": 0.19660811871290207,
      "layer": 0.09350453689694405,
      "mlm": 1.5831890404224396,
      "distill": 1.1181388795375824,
      "teacher_acc": 0.7138730883598328
    },
    {
      "step": 6481,
      "lr": 3.348739310341068e-05,
      "elapsed_s": 4815.320013046265,
      "loss": 3.100082576274872,
      "hidden": 0.19121718034148216,
      "layer": 0.09285282902419567,
      "mlm": 1.5914396941661835,
      "distill": 1.0797822177410126,
      "teacher_acc": 0.7088670581579208
    },
    {
      "step": 6491,
      "lr": 3.344002006077487e-05,
      "elapsed_s": 4822.720309495926,
      "loss": 3.086649239063263,
      "hidden": 0.1941196583211422,
      "layer": 0.09270430542528629,
      "mlm": 1.5763840079307556,
      "distill": 1.0756738036870956,
      "teacher_acc": 0.7164540588855743
    },
    {
      "step": 6501,
      "lr": 3.339261278916512e-05,
      "elapsed_s": 4830.096178293228,
      "loss": 3.1435670256614685,
      "hidden": 0.1970505379140377,
      "layer": 0.09375018440186977,
      "mlm": 1.5892204344272614,
      "distill": 1.1133703738451004,
      "teacher_acc": 0.716537281870842
    },
    {
      "step": 6511,
      "lr": 3.3345171480844275e-05,
      "elapsed_s": 4837.47211265564,
      "loss": 3.0550175309181213,
      "hidden": 0.19547664374113083,
      "layer": 0.09262915141880512,
      "mlm": 1.5328003764152527,
      "distill": 1.084949404001236,
      "teacher_acc": 0.727649912238121
    },
    {
      "step": 6521,
      "lr": 3.329769632821321e-05,
      "elapsed_s": 4844.863247156143,
      "loss": 3.157544434070587,
      "hidden": 0.19395290315151215,
      "layer": 0.0920556802302599,
      "mlm": 1.5867230296134949,
      "distill": 1.1368877291679382,
      "teacher_acc": 0.7227001935243607
    },
    {
      "step": 6531,
      "lr": 3.3250187523810074e-05,
      "elapsed_s": 4852.246895074844,
      "loss": 3.0899821519851685,
      "hidden": 0.19362366572022438,
      "layer": 0.09250318631529808,
      "mlm": 1.562640905380249,
      "distill": 1.0938422977924347,
      "teacher_acc": 0.7261867374181747
    },
    {
      "step": 6541,
      "lr": 3.3202645260309484e-05,
      "elapsed_s": 4859.687425136566,
      "loss": 2.960499405860901,
      "hidden": 0.19107167795300484,
      "layer": 0.09141702950000763,
      "mlm": 1.4866342842578888,
      "distill": 1.046013206243515,
      "teacher_acc": 0.7326999008655548
    },
    {
      "step": 6551,
      "lr": 3.3155069730521735e-05,
      "elapsed_s": 4867.129043340683,
      "loss": 3.1678892374038696,
      "hidden": 0.19265920296311378,
      "layer": 0.09390268661081791,
      "mlm": 1.592348963022232,
      "distill": 1.1432705223560333,
      "teacher_acc": 0.7236759662628174
    },
    {
      "step": 6561,
      "lr": 3.310746112739207e-05,
      "elapsed_s": 4874.547936201096,
      "loss": 2.8335387110710144,
      "hidden": 0.18653929233551025,
      "layer": 0.09028731659054756,
      "mlm": 1.4007984399795532,
      "distill": 1.0145180076360703,
      "teacher_acc": 0.7442683577537537
    },
    {
      "step": 6571,
      "lr": 3.305981964399984e-05,
      "elapsed_s": 4881.953952074051,
      "loss": 2.99005126953125,
      "hidden": 0.19211777672171593,
      "layer": 0.0928332582116127,
      "mlm": 1.490359902381897,
      "distill": 1.069039136171341,
      "teacher_acc": 0.7327405512332916
    },
    {
      "step": 6581,
      "lr": 3.301214547355774e-05,
      "elapsed_s": 4889.365768194199,
      "loss": 2.935452938079834,
      "hidden": 0.1935550682246685,
      "layer": 0.09252864494919777,
      "mlm": 1.4317081272602081,
      "distill": 1.0703704059123993,
      "teacher_acc": 0.7416944652795792
    },
    {
      "step": 6591,
      "lr": 3.296443880941106e-05,
      "elapsed_s": 4896.7523238658905,
      "loss": 2.9808900952339172,
      "hidden": 0.18906664475798607,
      "layer": 0.09109768085181713,
      "mlm": 1.5382168889045715,
      "distill": 1.0189910978078842,
      "teacher_acc": 0.7174976319074631
    },
    {
      "step": 6601,
      "lr": 3.2916699845036816e-05,
      "elapsed_s": 4904.1211133003235,
      "loss": 3.0441697239875793,
      "hidden": 0.1883394420146942,
      "layer": 0.09216994792222977,
      "mlm": 1.5635958909988403,
      "distill": 1.057809978723526,
      "teacher_acc": 0.7177467197179794
    },
    {
      "step": 6611,
      "lr": 3.286892877404307e-05,
      "elapsed_s": 4911.519555091858,
      "loss": 3.128444194793701,
      "hidden": 0.19554823637008667,
      "layer": 0.09384314715862274,
      "mlm": 1.5746730864048004,
      "distill": 1.1157531142234802,
      "teacher_acc": 0.7204395979642868
    },
    {
      "step": 6621,
      "lr": 3.282112579016808e-05,
      "elapsed_s": 4918.904745101929,
      "loss": 3.121576249599457,
      "hidden": 0.1997637078166008,
      "layer": 0.09400556981563568,
      "mlm": 1.5162419974803925,
      "distill": 1.158804014325142,
      "teacher_acc": 0.7303462326526642
    },
    {
      "step": 6631,
      "lr": 3.277329108727951e-05,
      "elapsed_s": 4926.328701257706,
      "loss": 3.1097660660743713,
      "hidden": 0.189231526106596,
      "layer": 0.09152696095407009,
      "mlm": 1.6516403555870056,
      "distill": 1.0338991731405258,
      "teacher_acc": 0.7059909999370575
    },
    {
      "step": 6641,
      "lr": 3.272542485937369e-05,
      "elapsed_s": 4933.771844148636,
      "loss": 3.0863441824913025,
      "hidden": 0.19343487545847893,
      "layer": 0.09309600107371807,
      "mlm": 1.5646571815013885,
      "distill": 1.0882692486047745,
      "teacher_acc": 0.7150850892066956
    },
    {
      "step": 6651,
      "lr": 3.267752730057477e-05,
      "elapsed_s": 4941.220808506012,
      "loss": 3.063106656074524,
      "hidden": 0.18942636251449585,
      "layer": 0.09161435253918171,
      "mlm": 1.54872065782547,
      "distill": 1.0897260904312134,
      "teacher_acc": 0.7225798815488815
    },
    {
      "step": 6661,
      "lr": 3.2629598605133996e-05,
      "elapsed_s": 4948.6432929039,
      "loss": 3.113893747329712,
      "hidden": 0.193775475025177,
      "layer": 0.09305213578045368,
      "mlm": 1.5490150451660156,
      "distill": 1.1308017075061798,
      "teacher_acc": 0.7287840694189072
    },
    {
      "step": 6671,
      "lr": 3.258163896742888e-05,
      "elapsed_s": 4956.052638292313,
      "loss": 3.068621039390564,
      "hidden": 0.19501608982682228,
      "layer": 0.09242750890552998,
      "mlm": 1.5083639025688171,
      "distill": 1.1240111887454987,
      "teacher_acc": 0.7380156069993973
    },
    {
      "step": 6681,
      "lr": 3.2533648581962404e-05,
      "elapsed_s": 4963.4812223911285,
      "loss": 3.1689722537994385,
      "hidden": 0.19149839878082275,
      "layer": 0.09236111864447594,
      "mlm": 1.6389763951301575,
      "distill": 1.1008186042308807,
      "teacher_acc": 0.7034313976764679
    },
    {
      "step": 6691,
      "lr": 3.248562764336226e-05,
      "elapsed_s": 4970.899342775345,
      "loss": 3.0094494223594666,
      "hidden": 0.1891632340848446,
      "layer": 0.09222275018692017,
      "mlm": 1.475536733865738,
      "distill": 1.1094749569892883,
      "teacher_acc": 0.7446059137582779
    },
    {
      "step": 6701,
      "lr": 3.243757634638008e-05,
      "elapsed_s": 4978.2990798950195,
      "loss": 3.0460970997810364,
      "hidden": 0.19436076655983925,
      "layer": 0.0914185643196106,
      "mlm": 1.5273028016090393,
      "distill": 1.0843634903430939,
      "teacher_acc": 0.7241494357585907
    },
    {
      "step": 6711,
      "lr": 3.238949488589056e-05,
      "elapsed_s": 4985.680917739868,
      "loss": 2.995810627937317,
      "hidden": 0.19724280387163162,
      "layer": 0.09163709543645382,
      "mlm": 1.4350907802581787,
      "distill": 1.1204157173633575,
      "teacher_acc": 0.7484878897666931
    },
    {
      "step": 6721,
      "lr": 3.234138345689077e-05,
      "elapsed_s": 4993.111385822296,
      "loss": 3.1268614530563354,
      "hidden": 0.19544949010014534,
      "layer": 0.0933297649025917,
      "mlm": 1.5857884883880615,
      "distill": 1.1035090684890747,
      "teacher_acc": 0.726325511932373
    },
    {
      "step": 6731,
      "lr": 3.22932422544993e-05,
      "elapsed_s": 5000.533604383469,
      "loss": 3.1060794591903687,
      "hidden": 0.1927817091345787,
      "layer": 0.09361829981207848,
      "mlm": 1.5479795932769775,
      "distill": 1.1257272362709045,
      "teacher_acc": 0.7274075001478195
    },
    {
      "step": 6741,
      "lr": 3.2245071473955505e-05,
      "elapsed_s": 5007.960158586502,
      "loss": 3.088205575942993,
      "hidden": 0.19429773464798927,
      "layer": 0.09291867725551128,
      "mlm": 1.503915011882782,
      "distill": 1.1492358148097992,
      "teacher_acc": 0.7314756512641907
    },
    {
      "step": 6751,
      "lr": 3.2196871310618655e-05,
      "elapsed_s": 5015.370353460312,
      "loss": 3.070266008377075,
      "hidden": 0.18851029872894287,
      "layer": 0.09133115224540234,
      "mlm": 1.6062949001789093,
      "distill": 1.0412848889827728,
      "teacher_acc": 0.7115263342857361
    },
    {
      "step": 6761,
      "lr": 3.214864195996723e-05,
      "elapsed_s": 5022.797037124634,
      "loss": 3.345462739467621,
      "hidden": 0.20098154619336128,
      "layer": 0.095725879073143,
      "mlm": 1.651926040649414,
      "distill": 1.2437106668949127,
      "teacher_acc": 0.7080221474170685
    },
    {
      "step": 6771,
      "lr": 3.210038361759807e-05,
      "elapsed_s": 5030.223799467087,
      "loss": 3.1520798206329346,
      "hidden": 0.1933206506073475,
      "layer": 0.09275970607995987,
      "mlm": 1.6026111841201782,
      "distill": 1.1164474189281464,
      "teacher_acc": 0.7200828641653061
    },
    {
      "step": 6781,
      "lr": 3.2052096479225566e-05,
      "elapsed_s": 5037.619102716446,
      "loss": 3.073320508003235,
      "hidden": 0.1901359036564827,
      "layer": 0.09135800041258335,
      "mlm": 1.607835054397583,
      "distill": 1.0395347476005554,
      "teacher_acc": 0.702377662062645
    },
    {
      "step": 6791,
      "lr": 3.200378074068092e-05,
      "elapsed_s": 5045.027637481689,
      "loss": 2.9630327820777893,
      "hidden": 0.188953448086977,
      "layer": 0.08973542228341103,
      "mlm": 1.4481555819511414,
      "distill": 1.092102587223053,
      "teacher_acc": 0.7447509914636612
    },
    {
      "step": 6801,
      "lr": 3.195543659791132e-05,
      "elapsed_s": 5052.455496072769,
      "loss": 3.184573531150818,
      "hidden": 0.19670430198311806,
      "layer": 0.09374848380684853,
      "mlm": 1.598709374666214,
      "distill": 1.1455812454223633,
      "teacher_acc": 0.7238747477531433
    },
    {
      "step": 6811,
      "lr": 3.1907064246979134e-05,
      "elapsed_s": 5059.8716452121735,
      "loss": 3.1014814376831055,
      "hidden": 0.198008444160223,
      "layer": 0.09327641502022743,
      "mlm": 1.526116669178009,
      "distill": 1.13270965218544,
      "teacher_acc": 0.7400634735822678
    },
    {
      "step": 6821,
      "lr": 3.185866388406115e-05,
      "elapsed_s": 5067.277510404587,
      "loss": 3.0280473232269287,
      "hidden": 0.1882462352514267,
      "layer": 0.09147290326654911,
      "mlm": 1.5396287739276886,
      "distill": 1.0661896169185638,
      "teacher_acc": 0.7172193676233292
    },
    {
      "step": 6831,
      "lr": 3.1810235705447736e-05,
      "elapsed_s": 5074.721405267715,
      "loss": 3.0405390858650208,
      "hidden": 0.18884649127721786,
      "layer": 0.09161466173827648,
      "mlm": 1.5616739392280579,
      "distill": 1.0553647875785828,
      "teacher_acc": 0.7242964059114456
    },
    {
      "step": 6841,
      "lr": 3.17617799075421e-05,
      "elapsed_s": 5082.147145748138,
      "loss": 3.150662899017334,
      "hidden": 0.19792652130126953,
      "layer": 0.09250216372311115,
      "mlm": 1.5923795104026794,
      "distill": 1.1161792576313019,
      "teacher_acc": 0.7144245207309723
    },
    {
      "step": 6851,
      "lr": 3.1713296686859426e-05,
      "elapsed_s": 5089.5581521987915,
      "loss": 3.0511292219161987,
      "hidden": 0.19369656592607498,
      "layer": 0.09194767847657204,
      "mlm": 1.5319861769676208,
      "distill": 1.0857761055231094,
      "teacher_acc": 0.722461998462677
    },
    {
      "step": 6861,
      "lr": 3.1664786240026136e-05,
      "elapsed_s": 5096.961491107941,
      "loss": 3.1119178533554077,
      "hidden": 0.19246786460280418,
      "layer": 0.093105249106884,
      "mlm": 1.5724062025547028,
      "distill": 1.1080233305692673,
      "teacher_acc": 0.7188770174980164
    },
    {
      "step": 6871,
      "lr": 3.161624876377906e-05,
      "elapsed_s": 5104.37891125679,
      "loss": 2.998623847961426,
      "hidden": 0.1897561028599739,
      "layer": 0.09041817486286163,
      "mlm": 1.5298241972923279,
      "distill": 1.0440782606601715,
      "teacher_acc": 0.723378375172615
    },
    {
      "step": 6881,
      "lr": 3.1567684454964675e-05,
      "elapsed_s": 5111.760212421417,
      "loss": 3.081710457801819,
      "hidden": 0.1989128142595291,
      "layer": 0.09478523023426533,
      "mlm": 1.5313127040863037,
      "distill": 1.105179488658905,
      "teacher_acc": 0.7259162366390228
    },
    {
      "step": 6891,
      "lr": 3.151909351053823e-05,
      "elapsed_s": 5119.181850194931,
      "loss": 3.1680506467819214,
      "hidden": 0.19792239367961884,
      "layer": 0.09328272752463818,
      "mlm": 1.6033442616462708,
      "distill": 1.1222202479839325,
      "teacher_acc": 0.7137023061513901
    },
    {
      "step": 6901,
      "lr": 3.147047612756302e-05,
      "elapsed_s": 5126.566792726517,
      "loss": 2.959702789783478,
      "hidden": 0.18608924001455307,
      "layer": 0.08934372663497925,
      "mlm": 1.5162666141986847,
      "distill": 1.0265858322381973,
      "teacher_acc": 0.7216834425926208
    },
    {
      "step": 6911,
      "lr": 3.142183250320957e-05,
      "elapsed_s": 5133.949009418488,
      "loss": 3.019935131072998,
      "hidden": 0.19341742992401123,
      "layer": 0.09184035658836365,
      "mlm": 1.5181127786636353,
      "distill": 1.069067269563675,
      "teacher_acc": 0.7371430695056915
    },
    {
      "step": 6921,
      "lr": 3.1373162834754834e-05,
      "elapsed_s": 5141.373652458191,
      "loss": 2.9760284423828125,
      "hidden": 0.18533772975206375,
      "layer": 0.09004099853336811,
      "mlm": 1.5557239651679993,
      "distill": 1.0046085268259048,
      "teacher_acc": 0.7127281129360199
    },
    {
      "step": 6931,
      "lr": 3.1324467319581355e-05,
      "elapsed_s": 5148.793612003326,
      "loss": 2.9915781021118164,
      "hidden": 0.18876386806368828,
      "layer": 0.09078200161457062,
      "mlm": 1.529301106929779,
      "distill": 1.0393582582473755,
      "teacher_acc": 0.721349835395813
    },
    {
      "step": 6941,
      "lr": 3.1275746155176546e-05,
      "elapsed_s": 5156.1771676540375,
      "loss": 3.071367561817169,
      "hidden": 0.19733679667115211,
      "layer": 0.0927790105342865,
      "mlm": 1.520515352487564,
      "distill": 1.109789103269577,
      "teacher_acc": 0.7285658717155457
    },
    {
      "step": 6951,
      "lr": 3.12269995391318e-05,
      "elapsed_s": 5163.591950416565,
      "loss": 3.040101647377014,
      "hidden": 0.19054826349020004,
      "layer": 0.09138341248035431,
      "mlm": 1.4911785125732422,
      "distill": 1.1221349835395813,
      "teacher_acc": 0.7426030337810516
    },
    {
      "step": 6961,
      "lr": 3.1178227669141744e-05,
      "elapsed_s": 5170.970824480057,
      "loss": 3.2094080448150635,
      "hidden": 0.19569090753793716,
      "layer": 0.0930653065443039,
      "mlm": 1.6358653903007507,
      "distill": 1.1356281042099,
      "teacher_acc": 0.7002932280302048
    },
    {
      "step": 6971,
      "lr": 3.112943074300344e-05,
      "elapsed_s": 5178.367203235626,
      "loss": 3.0443124771118164,
      "hidden": 0.18877273052930832,
      "layer": 0.09133052453398705,
      "mlm": 1.568383514881134,
      "distill": 1.0527182072401047,
      "teacher_acc": 0.7202931046485901
    },
    {
      "step": 6981,
      "lr": 3.108060895861555e-05,
      "elapsed_s": 5185.790980100632,
      "loss": 3.1210193037986755,
      "hidden": 0.19748305901885033,
      "layer": 0.09336615353822708,
      "mlm": 1.5967898964881897,
      "distill": 1.082580327987671,
      "teacher_acc": 0.7124812453985214
    },
    {
      "step": 6991,
      "lr": 3.103176251397756e-05,
      "elapsed_s": 5193.194725036621,
      "loss": 3.1837591528892517,
      "hidden": 0.19740600511431694,
      "layer": 0.09489451162517071,
      "mlm": 1.613466054201126,
      "distill": 1.1280338615179062,
      "teacher_acc": 0.7135652899742126
    },
    {
      "step": 7001,
      "lr": 3.098289160718895e-05,
      "elapsed_s": 5202.589289188385,
      "loss": 3.0009286999702454,
      "hidden": 0.19027046486735344,
      "layer": 0.09135977365076542,
      "mlm": 1.4727427065372467,
      "distill": 1.1019651293754578,
      "teacher_acc": 0.7417635917663574
    },
    {
      "step": 7011,
      "lr": 3.093399643644842e-05,
      "elapsed_s": 5209.9331138134,
      "loss": 3.109593629837036,
      "hidden": 0.19878026843070984,
      "layer": 0.09495226666331291,
      "mlm": 1.4822051525115967,
      "distill": 1.1823517978191376,
      "teacher_acc": 0.7377219349145889
    },
    {
      "step": 7021,
      "lr": 3.0885077200053086e-05,
      "elapsed_s": 5217.281810283661,
      "loss": 3.0813854932785034,
      "hidden": 0.18865623697638512,
      "layer": 0.09211434796452522,
      "mlm": 1.6016297340393066,
      "distill": 1.0563861429691315,
      "teacher_acc": 0.7121593803167343
    },
    {
      "step": 7031,
      "lr": 3.083613409639764e-05,
      "elapsed_s": 5224.636778831482,
      "loss": 3.027763605117798,
      "hidden": 0.19090311601758003,
      "layer": 0.09206153452396393,
      "mlm": 1.5396234691143036,
      "distill": 1.0603031814098358,
      "teacher_acc": 0.7226303815841675
    },
    {
      "step": 7041,
      "lr": 3.078716732397358e-05,
      "elapsed_s": 5232.017203569412,
      "loss": 3.113310396671295,
      "hidden": 0.1947733648121357,
      "layer": 0.09276142157614231,
      "mlm": 1.592847317457199,
      "distill": 1.0845356285572052,
      "teacher_acc": 0.7032542377710342
    },
    {
      "step": 7051,
      "lr": 3.073817708136839e-05,
      "elapsed_s": 5239.434553146362,
      "loss": 2.982120990753174,
      "hidden": 0.1879071556031704,
      "layer": 0.09047316014766693,
      "mlm": 1.508483111858368,
      "distill": 1.052587017416954,
      "teacher_acc": 0.7232035845518112
    },
    {
      "step": 7061,
      "lr": 3.068916356726475e-05,
      "elapsed_s": 5246.876905679703,
      "loss": 3.029772698879242,
      "hidden": 0.19194410368800163,
      "layer": 0.09143619798123837,
      "mlm": 1.5159733593463898,
      "distill": 1.084193080663681,
      "teacher_acc": 0.7379245758056641
    },
    {
      "step": 7071,
      "lr": 3.064012698043972e-05,
      "elapsed_s": 5254.292939186096,
      "loss": 2.9173487424850464,
      "hidden": 0.18981150910258293,
      "layer": 0.09117615409195423,
      "mlm": 1.4385906457901,
      "distill": 1.0535470694303513,
      "teacher_acc": 0.7415085732936859
    },
    {
      "step": 7081,
      "lr": 3.0591067519763895e-05,
      "elapsed_s": 5261.71030497551,
      "loss": 3.086690664291382,
      "hidden": 0.1918393149971962,
      "layer": 0.09144704043865204,
      "mlm": 1.5868307948112488,
      "distill": 1.0704577267169952,
      "teacher_acc": 0.7155803143978119
    },
    {
      "step": 7091,
      "lr": 3.0541985384200686e-05,
      "elapsed_s": 5269.148916959763,
      "loss": 3.0205820202827454,
      "hidden": 0.19150079414248466,
      "layer": 0.09195362590253353,
      "mlm": 1.5693199336528778,
      "distill": 1.0222836285829544,
      "teacher_acc": 0.7204276323318481
    },
    {
      "step": 7101,
      "lr": 3.0492880772805433e-05,
      "elapsed_s": 5276.561764001846,
      "loss": 3.089262306690216,
      "hidden": 0.19368674978613853,
      "layer": 0.09256312064826488,
      "mlm": 1.5087122917175293,
      "distill": 1.1468949019908905,
      "teacher_acc": 0.7371128350496292
    },
    {
      "step": 7111,
      "lr": 3.044375388472464e-05,
      "elapsed_s": 5283.994080543518,
      "loss": 2.975350260734558,
      "hidden": 0.18741286545991898,
      "layer": 0.0907050184905529,
      "mlm": 1.5497384369373322,
      "distill": 1.005433663725853,
      "teacher_acc": 0.7283224910497665
    },
    {
      "step": 7121,
      "lr": 3.0394604919195156e-05,
      "elapsed_s": 5291.404820919037,
      "loss": 3.03240966796875,
      "hidden": 0.18648329004645348,
      "layer": 0.09007862396538258,
      "mlm": 1.5395373702049255,
      "distill": 1.074866384267807,
      "teacher_acc": 0.7247832864522934
    },
    {
      "step": 7131,
      "lr": 3.0345434075543354e-05,
      "elapsed_s": 5298.832231998444,
      "loss": 2.946578860282898,
      "hidden": 0.1795550175011158,
      "layer": 0.08789872750639915,
      "mlm": 1.5550656914710999,
      "distill": 0.9884537607431412,
      "teacher_acc": 0.7110612243413925
    },
    {
      "step": 7141,
      "lr": 3.0296241553184362e-05,
      "elapsed_s": 5306.259646654129,
      "loss": 3.0904070138931274,
      "hidden": 0.1931934393942356,
      "layer": 0.09335710667073727,
      "mlm": 1.5571808218955994,
      "distill": 1.1001607775688171,
      "teacher_acc": 0.7159523516893387
    },
    {
      "step": 7151,
      "lr": 3.024702755162119e-05,
      "elapsed_s": 5313.671886205673,
      "loss": 3.080600678920746,
      "hidden": 0.18544824421405792,
      "layer": 0.09086282178759575,
      "mlm": 1.5931247174739838,
      "distill": 1.071148008108139,
      "teacher_acc": 0.7137468606233597
    },
    {
      "step": 7161,
      "lr": 3.0197792270443982e-05,
      "elapsed_s": 5321.093729496002,
      "loss": 2.995771586894989,
      "hidden": 0.18957290798425674,
      "layer": 0.09223783202469349,
      "mlm": 1.4757322072982788,
      "distill": 1.0947747081518173,
      "teacher_acc": 0.7404986172914505
    },
    {
      "step": 7171,
      "lr": 3.0148535909329185e-05,
      "elapsed_s": 5328.505641937256,
      "loss": 3.1087055802345276,
      "hidden": 0.20082856714725494,
      "layer": 0.0950765497982502,
      "mlm": 1.479261964559555,
      "distill": 1.180248200893402,
      "teacher_acc": 0.742479607462883
    },
    {
      "step": 7181,
      "lr": 3.009925866803872e-05,
      "elapsed_s": 5335.9043226242065,
      "loss": 3.051198720932007,
      "hidden": 0.19042016565799713,
      "layer": 0.09104670397937298,
      "mlm": 1.5633728206157684,
      "distill": 1.0614622235298157,
      "teacher_acc": 0.7152037024497986
    },
    {
      "step": 7191,
      "lr": 3.0049960746419205e-05,
      "elapsed_s": 5343.29391336441,
      "loss": 3.1746583580970764,
      "hidden": 0.20141048729419708,
      "layer": 0.09385591745376587,
      "mlm": 1.551397442817688,
      "distill": 1.1735118925571442,
      "teacher_acc": 0.7325659543275833
    },
    {
      "step": 7201,
      "lr": 3.0000642344401113e-05,
      "elapsed_s": 5350.691730976105,
      "loss": 3.0568103194236755,
      "hidden": 0.1941068433225155,
      "layer": 0.09366768971085548,
      "mlm": 1.495560646057129,
      "distill": 1.1262021362781525,
      "teacher_acc": 0.7357208132743835
    },
    {
      "step": 7211,
      "lr": 2.9951303661997994e-05,
      "elapsed_s": 5358.096495389938,
      "loss": 3.0217697620391846,
      "hidden": 0.1900668479502201,
      "layer": 0.09107355959713459,
      "mlm": 1.5312622487545013,
      "distill": 1.064837023615837,
      "teacher_acc": 0.716829776763916
    },
    {
      "step": 7221,
      "lr": 2.9901944899305634e-05,
      "elapsed_s": 5365.534209728241,
      "loss": 3.041344165802002,
      "hidden": 0.1867748312652111,
      "layer": 0.09121344797313213,
      "mlm": 1.5269192457199097,
      "distill": 1.0952685177326202,
      "teacher_acc": 0.7284190654754639
    },
    {
      "step": 7231,
      "lr": 2.985256625650125e-05,
      "elapsed_s": 5372.942424058914,
      "loss": 3.056334376335144,
      "hidden": 0.18949979916214943,
      "layer": 0.09091497026383877,
      "mlm": 1.5422196686267853,
      "distill": 1.089657574892044,
      "teacher_acc": 0.7233892679214478
    },
    {
      "step": 7241,
      "lr": 2.9803167933842714e-05,
      "elapsed_s": 5380.352122306824,
      "loss": 3.096451222896576,
      "hidden": 0.19823025912046432,
      "layer": 0.09320971556007862,
      "mlm": 1.506306141614914,
      "distill": 1.1470797061920166,
      "teacher_acc": 0.7353167086839676
    },
    {
      "step": 7251,
      "lr": 2.975375013166767e-05,
      "elapsed_s": 5387.792006969452,
      "loss": 3.0039161443710327,
      "hidden": 0.1897176057100296,
      "layer": 0.09119299799203873,
      "mlm": 1.4622952342033386,
      "distill": 1.116589218378067,
      "teacher_acc": 0.7452525645494461
    },
    {
      "step": 7261,
      "lr": 2.9704313050392784e-05,
      "elapsed_s": 5395.225994110107,
      "loss": 3.038465201854706,
      "hidden": 0.1900322064757347,
      "layer": 0.09257050603628159,
      "mlm": 1.5043206214904785,
      "distill": 1.1077949106693268,
      "teacher_acc": 0.737324059009552
    },
    {
      "step": 7271,
      "lr": 2.9654856890512923e-05,
      "elapsed_s": 5402.645349264145,
      "loss": 3.0819523334503174,
      "hidden": 0.1897885426878929,
      "layer": 0.09236268512904644,
      "mlm": 1.522098958492279,
      "distill": 1.1340949833393097,
      "teacher_acc": 0.7331811189651489
    },
    {
      "step": 7281,
      "lr": 2.960538185260029e-05,
      "elapsed_s": 5410.090573787689,
      "loss": 2.996766448020935,
      "hidden": 0.18811232969164848,
      "layer": 0.09148579835891724,
      "mlm": 1.477211058139801,
      "distill": 1.097587689757347,
      "teacher_acc": 0.7354803681373596
    },
    {
      "step": 7291,
      "lr": 2.9555888137303695e-05,
      "elapsed_s": 5417.5196833610535,
      "loss": 3.0930901765823364,
      "hidden": 0.1921503245830536,
      "layer": 0.09215942770242691,
      "mlm": 1.5601767599582672,
      "distill": 1.102532982826233,
      "teacher_acc": 0.7198748290538788
    },
    {
      "step": 7301,
      "lr": 2.950637594534765e-05,
      "elapsed_s": 5424.924043893814,
      "loss": 3.108459174633026,
      "hidden": 0.1888951063156128,
      "layer": 0.09175702184438705,
      "mlm": 1.5939758121967316,
      "distill": 1.0908145904541016,
      "teacher_acc": 0.7066957950592041
    },
    {
      "step": 7311,
      "lr": 2.945684547753163e-05,
      "elapsed_s": 5432.341083049774,
      "loss": 2.9740148782730103,
      "hidden": 0.18854829296469688,
      "layer": 0.0911153145134449,
      "mlm": 1.476170301437378,
      "distill": 1.0751902759075165,
      "teacher_acc": 0.7420265078544617
    },
    {
      "step": 7321,
      "lr": 2.9407296934729227e-05,
      "elapsed_s": 5439.782258987427,
      "loss": 2.996560573577881,
      "hidden": 0.1884702518582344,
      "layer": 0.09181415103375912,
      "mlm": 1.4930247068405151,
      "distill": 1.0806882977485657,
      "teacher_acc": 0.7325174659490585
    },
    {
      "step": 7331,
      "lr": 2.9357730517887322e-05,
      "elapsed_s": 5447.193865299225,
      "loss": 3.046721339225769,
      "hidden": 0.19253314286470413,
      "layer": 0.09223195165395737,
      "mlm": 1.5545329451560974,
      "distill": 1.0610061883926392,
      "teacher_acc": 0.719712033867836
    },
    {
      "step": 7341,
      "lr": 2.9308146428025302e-05,
      "elapsed_s": 5454.618585109711,
      "loss": 3.1813237071037292,
      "hidden": 0.19184523448348045,
      "layer": 0.09340518340468407,
      "mlm": 1.678251564502716,
      "distill": 1.0726790875196457,
      "teacher_acc": 0.6962525099515915
    },
    {
      "step": 7351,
      "lr": 2.9258544866234207e-05,
      "elapsed_s": 5462.034168958664,
      "loss": 2.9647884368896484,
      "hidden": 0.18336690217256546,
      "layer": 0.08902732655405998,
      "mlm": 1.5431723594665527,
      "distill": 1.0103686302900314,
      "teacher_acc": 0.7187994122505188
    },
    {
      "step": 7361,
      "lr": 2.920892603367596e-05,
      "elapsed_s": 5469.460248947144,
      "loss": 2.9791354537010193,
      "hidden": 0.1860056333243847,
      "layer": 0.09013870172202587,
      "mlm": 1.5175555646419525,
      "distill": 1.0444992780685425,
      "teacher_acc": 0.7327378839254379
    },
    {
      "step": 7371,
      "lr": 2.9159290131582516e-05,
      "elapsed_s": 5476.882873773575,
      "loss": 3.1368666291236877,
      "hidden": 0.19810686260461807,
      "layer": 0.09451708942651749,
      "mlm": 1.5704328417778015,
      "distill": 1.122961550951004,
      "teacher_acc": 0.7244236320257187
    },
    {
      "step": 7381,
      "lr": 2.910963736125505e-05,
      "elapsed_s": 5484.295904159546,
      "loss": 3.060743808746338,
      "hidden": 0.1894250474870205,
      "layer": 0.09119093976914883,
      "mlm": 1.543716698884964,
      "distill": 1.0925814807415009,
      "teacher_acc": 0.7256015241146088
    },
    {
      "step": 7391,
      "lr": 2.9059967924063153e-05,
      "elapsed_s": 5491.7075164318085,
      "loss": 3.113831579685211,
      "hidden": 0.1924474872648716,
      "layer": 0.0932778399437666,
      "mlm": 1.5062773823738098,
      "distill": 1.1760202944278717,
      "teacher_acc": 0.7308279871940613
    },
    {
      "step": 7401,
      "lr": 2.9010282021444008e-05,
      "elapsed_s": 5499.116909980774,
      "loss": 3.1071866154670715,
      "hidden": 0.1968911774456501,
      "layer": 0.09397237747907639,
      "mlm": 1.5194157660007477,
      "distill": 1.1470022797584534,
      "teacher_acc": 0.7352503687143326
    },
    {
      "step": 7411,
      "lr": 2.896057985490157e-05,
      "elapsed_s": 5506.489717721939,
      "loss": 2.9479286074638367,
      "hidden": 0.1880086101591587,
      "layer": 0.08926203474402428,
      "mlm": 1.5025215446949005,
      "distill": 1.0247588455677032,
      "teacher_acc": 0.7241243571043015
    },
    {
      "step": 7421,
      "lr": 2.8910861626005776e-05,
      "elapsed_s": 5513.922233819962,
      "loss": 3.0811870098114014,
      "hidden": 0.1833505742251873,
      "layer": 0.09057346731424332,
      "mlm": 1.5416740477085114,
      "distill": 1.1275250911712646,
      "teacher_acc": 0.7313091605901718
    },
    {
      "step": 7431,
      "lr": 2.8861127536391662e-05,
      "elapsed_s": 5521.347511053085,
      "loss": 3.0270999670028687,
      "hidden": 0.18751073628664017,
      "layer": 0.09017894044518471,
      "mlm": 1.5574962198734283,
      "distill": 1.0494927763938904,
      "teacher_acc": 0.7185288965702057
    },
    {
      "step": 7441,
      "lr": 2.8811377787758636e-05,
      "elapsed_s": 5528.781310081482,
      "loss": 2.944574534893036,
      "hidden": 0.18232368305325508,
      "layer": 0.09029307216405869,
      "mlm": 1.5045892298221588,
      "distill": 1.0301914662122726,
      "teacher_acc": 0.723953053355217
    },
    {
      "step": 7451,
      "lr": 2.876161258186958e-05,
      "elapsed_s": 5536.15138053894,
      "loss": 3.1556102633476257,
      "hidden": 0.19513849914073944,
      "layer": 0.09267156943678856,
      "mlm": 1.5615369379520416,
      "distill": 1.1574605405330658,
      "teacher_acc": 0.7219764143228531
    },
    {
      "step": 7461,
      "lr": 2.8711832120550064e-05,
      "elapsed_s": 5543.534348487854,
      "loss": 2.988038420677185,
      "hidden": 0.19202617928385735,
      "layer": 0.09188261441886425,
      "mlm": 1.5007283985614777,
      "distill": 1.0573163330554962,
      "teacher_acc": 0.7287907600402832
    },
    {
      "step": 7471,
      "lr": 2.8662036605687554e-05,
      "elapsed_s": 5550.948871850967,
      "loss": 2.9269248247146606,
      "hidden": 0.18331390619277954,
      "layer": 0.08973966538906097,
      "mlm": 1.4979069232940674,
      "distill": 1.0175202935934067,
      "teacher_acc": 0.7278178185224533
    },
    {
      "step": 7481,
      "lr": 2.8612226239230532e-05,
      "elapsed_s": 5558.367915391922,
      "loss": 2.9800788164138794,
      "hidden": 0.1884198561310768,
      "layer": 0.09018511697649956,
      "mlm": 1.4672073125839233,
      "distill": 1.0909392833709717,
      "teacher_acc": 0.7386022508144379
    },
    {
      "step": 7491,
      "lr": 2.8562401223187746e-05,
      "elapsed_s": 5565.77348446846,
      "loss": 3.1249510645866394,
      "hidden": 0.19266561791300774,
      "layer": 0.09199255704879761,
      "mlm": 1.5972262620925903,
      "distill": 1.0963972061872482,
      "teacher_acc": 0.7165757715702057
    },
    {
      "step": 7501,
      "lr": 2.851256175962732e-05,
      "elapsed_s": 5573.217665672302,
      "loss": 2.995164930820465,
      "hidden": 0.18643347173929214,
      "layer": 0.09018766134977341,
      "mlm": 1.5648920238018036,
      "distill": 1.0123121440410614,
      "teacher_acc": 0.714265301823616
    },
    {
      "step": 7511,
      "lr": 2.8462708050676002e-05,
      "elapsed_s": 5580.635922431946,
      "loss": 2.9256468415260315,
      "hidden": 0.18669718503952026,
      "layer": 0.09063789062201977,
      "mlm": 1.4699408113956451,
      "distill": 1.0369927287101746,
      "teacher_acc": 0.7306879311800003
    },
    {
      "step": 7521,
      "lr": 2.8412840298518296e-05,
      "elapsed_s": 5588.084744215012,
      "loss": 2.919109284877777,
      "hidden": 0.19032857194542885,
      "layer": 0.09216167777776718,
      "mlm": 1.4353613555431366,
      "distill": 1.0570099353790283,
      "teacher_acc": 0.7390284091234207
    },
    {
      "step": 7531,
      "lr": 2.836295870539566e-05,
      "elapsed_s": 5595.527212142944,
      "loss": 2.955470621585846,
      "hidden": 0.1836264282464981,
      "layer": 0.08975952677428722,
      "mlm": 1.4952533841133118,
      "distill": 1.048084557056427,
      "teacher_acc": 0.7302305102348328
    },
    {
      "step": 7541,
      "lr": 2.8313063473605693e-05,
      "elapsed_s": 5602.948579072952,
      "loss": 3.038730561733246,
      "hidden": 0.18832578882575035,
      "layer": 0.09172491170465946,
      "mlm": 1.5492113530635834,
      "distill": 1.0670051872730255,
      "teacher_acc": 0.7130984663963318
    },
    {
      "step": 7551,
      "lr": 2.8263154805501297e-05,
      "elapsed_s": 5610.368504524231,
      "loss": 3.0298629999160767,
      "hidden": 0.18473528698086739,
      "layer": 0.09005849435925484,
      "mlm": 1.5272947549819946,
      "distill": 1.0880683660507202,
      "teacher_acc": 0.7283396422863007
    },
    {
      "step": 7561,
      "lr": 2.8213232903489865e-05,
      "elapsed_s": 5617.779463768005,
      "loss": 2.93791264295578,
      "hidden": 0.18626222014427185,
      "layer": 0.0905433613806963,
      "mlm": 1.484918624162674,
      "distill": 1.0351979285478592,
      "teacher_acc": 0.72165347635746
    },
    {
      "step": 7571,
      "lr": 2.8163297970032486e-05,
      "elapsed_s": 5625.190071821213,
      "loss": 2.999720811843872,
      "hidden": 0.1854301132261753,
      "layer": 0.09011204913258553,
      "mlm": 1.528872400522232,
      "distill": 1.0549321323633194,
      "teacher_acc": 0.7186605632305145
    },
    {
      "step": 7581,
      "lr": 2.8113350207643057e-05,
      "elapsed_s": 5632.596405506134,
      "loss": 2.974813997745514,
      "hidden": 0.18493220582604408,
      "layer": 0.09083750285208225,
      "mlm": 1.5081714391708374,
      "distill": 1.0513594150543213,
      "teacher_acc": 0.7363905310630798
    },
    {
      "step": 7591,
      "lr": 2.8063389818887536e-05,
      "elapsed_s": 5640.012062072754,
      "loss": 2.9743377566337585,
      "hidden": 0.18739143386483192,
      "layer": 0.08994429185986519,
      "mlm": 1.4900241196155548,
      "distill": 1.0645586848258972,
      "teacher_acc": 0.727920725941658
    },
    {
      "step": 7601,
      "lr": 2.8013417006383076e-05,
      "elapsed_s": 5647.444077968597,
      "loss": 2.981498420238495,
      "hidden": 0.1897204890847206,
      "layer": 0.09059731103479862,
      "mlm": 1.512906700372696,
      "distill": 1.043852150440216,
      "teacher_acc": 0.7244731038808823
    },
    {
      "step": 7611,
      "lr": 2.7963431972797217e-05,
      "elapsed_s": 5654.838367938995,
      "loss": 2.9485782980918884,
      "hidden": 0.19410225376486778,
      "layer": 0.09177334979176521,
      "mlm": 1.4363177120685577,
      "distill": 1.07816943526268,
      "teacher_acc": 0.7505611926317215
    },
    {
      "step": 7621,
      "lr": 2.791343492084706e-05,
      "elapsed_s": 5662.25759768486,
      "loss": 3.025204658508301,
      "hidden": 0.18749916926026344,
      "layer": 0.09079484269022942,
      "mlm": 1.5489496886730194,
      "distill": 1.0558592081069946,
      "teacher_acc": 0.7167472690343857
    },
    {
      "step": 7631,
      "lr": 2.7863426053298443e-05,
      "elapsed_s": 5669.669215917587,
      "loss": 3.02725350856781,
      "hidden": 0.19259554147720337,
      "layer": 0.09250566735863686,
      "mlm": 1.5181101858615875,
      "distill": 1.0776993930339813,
      "teacher_acc": 0.7277562320232391
    },
    {
      "step": 7641,
      "lr": 2.781340557296514e-05,
      "elapsed_s": 5677.071400642395,
      "loss": 2.9722774028778076,
      "hidden": 0.182041946798563,
      "layer": 0.08988325111567974,
      "mlm": 1.5233929753303528,
      "distill": 1.0398589074611664,
      "teacher_acc": 0.7190651297569275
    },
    {
      "step": 7651,
      "lr": 2.7763373682708e-05,
      "elapsed_s": 5684.458467960358,
      "loss": 3.089523196220398,
      "hidden": 0.1892138607800007,
      "layer": 0.09187457151710987,
      "mlm": 1.5424006283283234,
      "distill": 1.122757613658905,
      "teacher_acc": 0.7311834245920181
    },
    {
      "step": 7661,
      "lr": 2.771333058543416e-05,
      "elapsed_s": 5691.872886896133,
      "loss": 2.995387315750122,
      "hidden": 0.19011952728033066,
      "layer": 0.09119386412203312,
      "mlm": 1.5562802255153656,
      "distill": 1.0132710635662079,
      "teacher_acc": 0.7098977267742157
    },
    {
      "step": 7671,
      "lr": 2.7663276484096207e-05,
      "elapsed_s": 5699.303705215454,
      "loss": 2.8965154886245728,
      "hidden": 0.1843484304845333,
      "layer": 0.08909644186496735,
      "mlm": 1.4668704867362976,
      "distill": 1.0163998901844025,
      "teacher_acc": 0.7363534420728683
    },
    {
      "step": 7681,
      "lr": 2.761321158169134e-05,
      "elapsed_s": 5706.749795436859,
      "loss": 3.0926347374916077,
      "hidden": 0.18979458138346672,
      "layer": 0.09131170809268951,
      "mlm": 1.5974887311458588,
      "distill": 1.0699010640382767,
      "teacher_acc": 0.7185785472393036
    },
    {
      "step": 7691,
      "lr": 2.7563136081260582e-05,
      "elapsed_s": 5714.136500120163,
      "loss": 2.9837409257888794,
      "hidden": 0.18773405998945236,
      "layer": 0.09011197276413441,
      "mlm": 1.5356006920337677,
      "distill": 1.0276160687208176,
      "teacher_acc": 0.7193203866481781
    },
    {
      "step": 7701,
      "lr": 2.751305018588793e-05,
      "elapsed_s": 5721.558113336563,
      "loss": 3.1053909063339233,
      "hidden": 0.1888025589287281,
      "layer": 0.09253804944455624,
      "mlm": 1.6056976914405823,
      "distill": 1.0758191347122192,
      "teacher_acc": 0.7100610733032227
    },
    {
      "step": 7711,
      "lr": 2.7462954098699534e-05,
      "elapsed_s": 5728.986226558685,
      "loss": 3.0779277086257935,
      "hidden": 0.20126652717590332,
      "layer": 0.0939593967050314,
      "mlm": 1.4819110929965973,
      "distill": 1.1465038061141968,
      "teacher_acc": 0.7420305013656616
    },
    {
      "step": 7721,
      "lr": 2.741284802286288e-05,
      "elapsed_s": 5736.395162820816,
      "loss": 2.9501550793647766,
      "hidden": 0.1876591332256794,
      "layer": 0.09038502722978592,
      "mlm": 1.468202918767929,
      "distill": 1.0614413768053055,
      "teacher_acc": 0.7381609529256821
    },
    {
      "step": 7731,
      "lr": 2.736273216158597e-05,
      "elapsed_s": 5743.801215648651,
      "loss": 2.9356051683425903,
      "hidden": 0.1837463080883026,
      "layer": 0.09037880413234234,
      "mlm": 1.4661276936531067,
      "distill": 1.0567954033613205,
      "teacher_acc": 0.7325938791036606
    },
    {
      "step": 7741,
      "lr": 2.731260671811648e-05,
      "elapsed_s": 5751.182297468185,
      "loss": 3.0258572101593018,
      "hidden": 0.18725182861089706,
      "layer": 0.09016008116304874,
      "mlm": 1.5631127655506134,
      "distill": 1.0431607514619827,
      "teacher_acc": 0.7235102951526642
    },
    {
      "step": 7751,
      "lr": 2.726247189574095e-05,
      "elapsed_s": 5758.559280633926,
      "loss": 2.832945704460144,
      "hidden": 0.1791805736720562,
      "layer": 0.08798815310001373,
      "mlm": 1.46523779630661,
      "distill": 0.9653526693582535,
      "teacher_acc": 0.7313100546598434
    },
    {
      "step": 7761,
      "lr": 2.7212327897783964e-05,
      "elapsed_s": 5765.917881011963,
      "loss": 3.1580533385276794,
      "hidden": 0.19244497269392014,
      "layer": 0.0940419677644968,
      "mlm": 1.6090206801891327,
      "distill": 1.1171217560768127,
      "teacher_acc": 0.7082802504301071
    },
    {
      "step": 7771,
      "lr": 2.7162174927607313e-05,
      "elapsed_s": 5773.285195589066,
      "loss": 2.943654716014862,
      "hidden": 0.18304169550538063,
      "layer": 0.08952835202217102,
      "mlm": 1.5208623111248016,
      "distill": 1.0119448453187943,
      "teacher_acc": 0.7203290611505508
    },
    {
      "step": 7781,
      "lr": 2.711201318860918e-05,
      "elapsed_s": 5780.6790289878845,
      "loss": 3.137402653694153,
      "hidden": 0.18749522417783737,
      "layer": 0.09220555610954762,
      "mlm": 1.6497911214828491,
      "distill": 1.0665183514356613,
      "teacher_acc": 0.6992761939764023
    },
    {
      "step": 7791,
      "lr": 2.7061842884223298e-05,
      "elapsed_s": 5788.098623991013,
      "loss": 3.0021497011184692,
      "hidden": 0.1877472922205925,
      "layer": 0.09061541594564915,
      "mlm": 1.5282130539417267,
      "distill": 1.0531344264745712,
      "teacher_acc": 0.7240987718105316
    },
    {
      "step": 7801,
      "lr": 2.7011664217918154e-05,
      "elapsed_s": 5795.523969173431,
      "loss": 3.000724971294403,
      "hidden": 0.18767402693629265,
      "layer": 0.0895928218960762,
      "mlm": 1.5062476694583893,
      "distill": 1.0743328779935837,
      "teacher_acc": 0.7256947606801987
    },
    {
      "step": 7811,
      "lr": 2.6961477393196126e-05,
      "elapsed_s": 5802.91160774231,
      "loss": 2.9824647903442383,
      "hidden": 0.1847437620162964,
      "layer": 0.08968407101929188,
      "mlm": 1.5046601295471191,
      "distill": 1.063475027680397,
      "teacher_acc": 0.7328593283891678
    },
    {
      "step": 7821,
      "lr": 2.691128261359271e-05,
      "elapsed_s": 5810.31370806694,
      "loss": 3.106727182865143,
      "hidden": 0.19383690878748894,
      "layer": 0.0913463644683361,
      "mlm": 1.5556681156158447,
      "distill": 1.1177120804786682,
      "teacher_acc": 0.7260778397321701
    },
    {
      "step": 7831,
      "lr": 2.6861080082675628e-05,
      "elapsed_s": 5817.710686922073,
      "loss": 2.9853397607803345,
      "hidden": 0.18875868991017342,
      "layer": 0.09134745970368385,
      "mlm": 1.494173526763916,
      "distill": 1.0679751336574554,
      "teacher_acc": 0.7298176437616348
    },
    {
      "step": 7841,
      "lr": 2.6810870004044063e-05,
      "elapsed_s": 5825.099767208099,
      "loss": 2.826568067073822,
      "hidden": 0.18371308594942093,
      "layer": 0.08906679600477219,
      "mlm": 1.3784719705581665,
      "distill": 1.036136507987976,
      "teacher_acc": 0.7488227039575577
    },
    {
      "step": 7851,
      "lr": 2.67606525813278e-05,
      "elapsed_s": 5832.521641492844,
      "loss": 3.0510438084602356,
      "hidden": 0.19090741872787476,
      "layer": 0.09133046306669712,
      "mlm": 1.5387413501739502,
      "distill": 1.0848223567008972,
      "teacher_acc": 0.7251254767179489
    },
    {
      "step": 7861,
      "lr": 2.6710428018186405e-05,
      "elapsed_s": 5839.932518720627,
      "loss": 2.999074161052704,
      "hidden": 0.18459400534629822,
      "layer": 0.09065555594861507,
      "mlm": 1.5309627950191498,
      "distill": 1.0535955429077148,
      "teacher_acc": 0.7284251600503922
    },
    {
      "step": 7871,
      "lr": 2.6660196518308417e-05,
      "elapsed_s": 5847.3681054115295,
      "loss": 2.9408315420150757,
      "hidden": 0.1881016232073307,
      "layer": 0.09084421023726463,
      "mlm": 1.4482860565185547,
      "distill": 1.070920154452324,
      "teacher_acc": 0.748839259147644
    },
    {
      "step": 7881,
      "lr": 2.6609958285410485e-05,
      "elapsed_s": 5854.807632684708,
      "loss": 3.0478105545043945,
      "hidden": 0.18967768177390099,
      "layer": 0.09145846217870712,
      "mlm": 1.552637755870819,
      "distill": 1.0700882375240326,
      "teacher_acc": 0.7214302867650986
    },
    {
      "step": 7891,
      "lr": 2.6559713523236583e-05,
      "elapsed_s": 5862.202819347382,
      "loss": 2.9649361968040466,
      "hidden": 0.17942819744348526,
      "layer": 0.08906545117497444,
      "mlm": 1.54500350356102,
      "distill": 1.0165435373783112,
      "teacher_acc": 0.7163782566785812
    },
    {
      "step": 7901,
      "lr": 2.6509462435557152e-05,
      "elapsed_s": 5869.605575323105,
      "loss": 2.9366695880889893,
      "hidden": 0.1798684112727642,
      "layer": 0.08826954290270805,
      "mlm": 1.5255488753318787,
      "distill": 1.0072491466999054,
      "teacher_acc": 0.7170481085777283
    },
    {
      "step": 7911,
      "lr": 2.6459205226168293e-05,
      "elapsed_s": 5877.024721860886,
      "loss": 2.873809039592743,
      "hidden": 0.17885932326316833,
      "layer": 0.0873555913567543,
      "mlm": 1.492055356502533,
      "distill": 0.9803572297096252,
      "teacher_acc": 0.7278437912464142
    },
    {
      "step": 7921,
      "lr": 2.6408942098890936e-05,
      "elapsed_s": 5884.407346487045,
      "loss": 2.9233238697052,
      "hidden": 0.18204207718372345,
      "layer": 0.08829397894442081,
      "mlm": 1.4539612531661987,
      "distill": 1.061131477355957,
      "teacher_acc": 0.7383329272270203
    },
    {
      "step": 7931,
      "lr": 2.6358673257570005e-05,
      "elapsed_s": 5891.8187737464905,
      "loss": 2.9973945021629333,
      "hidden": 0.1873900070786476,
      "layer": 0.08988468162715435,
      "mlm": 1.4832879900932312,
      "distill": 1.0943840742111206,
      "teacher_acc": 0.7461092621088028
    },
    {
      "step": 7941,
      "lr": 2.63083989060736e-05,
      "elapsed_s": 5899.20214509964,
      "loss": 2.9359678626060486,
      "hidden": 0.18285968527197838,
      "layer": 0.08933073282241821,
      "mlm": 1.5082913041114807,
      "distill": 1.017291858792305,
      "teacher_acc": 0.7320671230554581
    },
    {
      "step": 7951,
      "lr": 2.6258119248292168e-05,
      "elapsed_s": 5906.589183092117,
      "loss": 2.9894891381263733,
      "hidden": 0.18684188276529312,
      "layer": 0.08924199640750885,
      "mlm": 1.5283924043178558,
      "distill": 1.0427919030189514,
      "teacher_acc": 0.7254522293806076
    },
    {
      "step": 7961,
      "lr": 2.6207834488137677e-05,
      "elapsed_s": 5913.98761177063,
      "loss": 3.027670681476593,
      "hidden": 0.19101034849882126,
      "layer": 0.09038809314370155,
      "mlm": 1.5506428182125092,
      "distill": 1.0498132407665253,
      "teacher_acc": 0.7211106419563293
    },
    {
      "step": 7971,
      "lr": 2.6157544829542796e-05,
      "elapsed_s": 5921.378497838974,
      "loss": 2.957527756690979,
      "hidden": 0.19058693945407867,
      "layer": 0.09104156866669655,
      "mlm": 1.4386736750602722,
      "distill": 1.0921594053506851,
      "teacher_acc": 0.7376282960176468
    },
    {
      "step": 7981,
      "lr": 2.6107250476460037e-05,
      "elapsed_s": 5928.744864463806,
      "loss": 3.073564291000366,
      "hidden": 0.19139793515205383,
      "layer": 0.09154894389212132,
      "mlm": 1.5407046675682068,
      "distill": 1.104289323091507,
      "teacher_acc": 0.7353475540876389
    },
    {
      "step": 7991,
      "lr": 2.6056951632860976e-05,
      "elapsed_s": 5936.154704809189,
      "loss": 3.134876847267151,
      "hidden": 0.19001232460141182,
      "layer": 0.09071553684771061,
      "mlm": 1.6584240198135376,
      "distill": 1.051070362329483,
      "teacher_acc": 0.7109333276748657
    },
    {
      "step": 8001,
      "lr": 2.600664850273538e-05,
      "elapsed_s": 5945.570575237274,
      "loss": 2.943476974964142,
      "hidden": 0.18500447645783424,
      "layer": 0.08882680721580982,
      "mlm": 1.486168771982193,
      "distill": 1.0428857505321503,
      "teacher_acc": 0.73287732899189
    },
    {
      "step": 8011,
      "lr": 2.595634129009042e-05,
      "elapsed_s": 5952.920455217361,
      "loss": 2.945117950439453,
      "hidden": 0.18122515082359314,
      "layer": 0.0886444877833128,
      "mlm": 1.5174431502819061,
      "distill": 1.0209023356437683,
      "teacher_acc": 0.720693364739418
    },
    {
      "step": 8021,
      "lr": 2.590603019894981e-05,
      "elapsed_s": 5960.297090053558,
      "loss": 3.0003084540367126,
      "hidden": 0.18534817174077034,
      "layer": 0.08990722335875034,
      "mlm": 1.5343915522098541,
      "distill": 1.050266832113266,
      "teacher_acc": 0.7225951552391052
    },
    {
      "step": 8031,
      "lr": 2.5855715433352994e-05,
      "elapsed_s": 5967.717488765717,
      "loss": 2.88617867231369,
      "hidden": 0.18049783632159233,
      "layer": 0.08847105130553246,
      "mlm": 1.4895272850990295,
      "distill": 0.9914201200008392,
      "teacher_acc": 0.7252546399831772
    },
    {
      "step": 8041,
      "lr": 2.580539719735433e-05,
      "elapsed_s": 5975.134567975998,
      "loss": 2.9384419918060303,
      "hidden": 0.18474457412958145,
      "layer": 0.08918085880577564,
      "mlm": 1.48868328332901,
      "distill": 1.0356791466474533,
      "teacher_acc": 0.7294705808162689
    },
    {
      "step": 8051,
      "lr": 2.5755075695022224e-05,
      "elapsed_s": 5982.567674636841,
      "loss": 2.950936436653137,
      "hidden": 0.1855541728436947,
      "layer": 0.08908936381340027,
      "mlm": 1.5060684978961945,
      "distill": 1.0292149037122726,
      "teacher_acc": 0.7337286621332169
    },
    {
      "step": 8061,
      "lr": 2.5704751130438365e-05,
      "elapsed_s": 5990.005371570587,
      "loss": 3.024790346622467,
      "hidden": 0.18189985305070877,
      "layer": 0.08877922780811787,
      "mlm": 1.5554241836071014,
      "distill": 1.0611768066883087,
      "teacher_acc": 0.7132090181112289
    },
    {
      "step": 8071,
      "lr": 2.5654423707696833e-05,
      "elapsed_s": 5997.415748596191,
      "loss": 2.899522840976715,
      "hidden": 0.18285351246595383,
      "layer": 0.08935621753334999,
      "mlm": 1.5021441280841827,
      "distill": 0.9869936406612396,
      "teacher_acc": 0.7266732156276703
    },
    {
      "step": 8081,
      "lr": 2.5604093630903307e-05,
      "elapsed_s": 6004.863036632538,
      "loss": 2.9205490946769714,
      "hidden": 0.17830318212509155,
      "layer": 0.08855077065527439,
      "mlm": 1.5176686346530914,
      "distill": 1.0019986629486084,
      "teacher_acc": 0.7154610008001328
    },
    {
      "step": 8091,
      "lr": 2.5553761104174247e-05,
      "elapsed_s": 6012.247004032135,
      "loss": 2.871517777442932,
      "hidden": 0.18540479987859726,
      "layer": 0.08971286006271839,
      "mlm": 1.4624199867248535,
      "distill": 0.9934317469596863,
      "teacher_acc": 0.7362517714500427
    },
    {
      "step": 8101,
      "lr": 2.550342633163601e-05,
      "elapsed_s": 6019.658682823181,
      "loss": 2.9650700092315674,
      "hidden": 0.18772930279374123,
      "layer": 0.09178568981587887,
      "mlm": 1.484730064868927,
      "distill": 1.0589885413646698,
      "teacher_acc": 0.7409871518611908
    },
    {
      "step": 8111,
      "lr": 2.54530895174241e-05,
      "elapsed_s": 6027.053880691528,
      "loss": 2.8513808846473694,
      "hidden": 0.1870022863149643,
      "layer": 0.08962621912360191,
      "mlm": 1.4357998073101044,
      "distill": 0.9967633932828903,
      "teacher_acc": 0.7356698513031006
    },
    {
      "step": 8121,
      "lr": 2.5402750865682285e-05,
      "elapsed_s": 6034.447416782379,
      "loss": 2.996609628200531,
      "hidden": 0.18410099297761917,
      "layer": 0.08966064453125,
      "mlm": 1.5587752163410187,
      "distill": 1.0248021632432938,
      "teacher_acc": 0.7142862677574158
    },
    {
      "step": 8131,
      "lr": 2.535241058056178e-05,
      "elapsed_s": 6041.86639046669,
      "loss": 2.949724316596985,
      "hidden": 0.17665766552090645,
      "layer": 0.08814729377627373,
      "mlm": 1.5256448090076447,
      "distill": 1.0266906023025513,
      "teacher_acc": 0.7201394885778427
    },
    {
      "step": 8141,
      "lr": 2.5302068866220434e-05,
      "elapsed_s": 6049.281199216843,
      "loss": 2.9043164253234863,
      "hidden": 0.19125251471996307,
      "layer": 0.091706408187747,
      "mlm": 1.4239628612995148,
      "distill": 1.05199533700943,
      "teacher_acc": 0.7435572743415833
    },
    {
      "step": 8151,
      "lr": 2.5251725926821888e-05,
      "elapsed_s": 6056.662350654602,
      "loss": 2.9747021198272705,
      "hidden": 0.1802802011370659,
      "layer": 0.08784078806638718,
      "mlm": 1.5812806487083435,
      "distill": 0.9889407008886337,
      "teacher_acc": 0.6977644264698029
    },
    {
      "step": 8161,
      "lr": 2.5201381966534748e-05,
      "elapsed_s": 6064.076615333557,
      "loss": 2.921797513961792,
      "hidden": 0.1928202472627163,
      "layer": 0.09113491885364056,
      "mlm": 1.400384098291397,
      "distill": 1.0902054160833359,
      "teacher_acc": 0.7523217499256134
    },
    {
      "step": 8171,
      "lr": 2.5151037189531772e-05,
      "elapsed_s": 6071.50076341629,
      "loss": 3.0105064511299133,
      "hidden": 0.17669742554426193,
      "layer": 0.08835393935441971,
      "mlm": 1.5948884785175323,
      "distill": 1.0180462449789047,
      "teacher_acc": 0.6993189305067062
    },
    {
      "step": 8181,
      "lr": 2.5100691799989012e-05,
      "elapsed_s": 6078.930648803711,
      "loss": 2.9743165373802185,
      "hidden": 0.1839786097407341,
      "layer": 0.08974463306367397,
      "mlm": 1.5268478095531464,
      "distill": 1.0346391648054123,
      "teacher_acc": 0.7220877557992935
    },
    {
      "step": 8191,
      "lr": 2.5050346002085028e-05,
      "elapsed_s": 6086.32984495163,
      "loss": 2.96433824300766,
      "hidden": 0.1838577762246132,
      "layer": 0.08956133760511875,
      "mlm": 1.5015293061733246,
      "distill": 1.050312653183937,
      "teacher_acc": 0.7331461906433105
    },
    {
      "step": 8201,
      "lr": 2.5e-05,
      "elapsed_s": 6093.723961830139,
      "loss": 2.9922373294830322,
      "hidden": 0.18583258241415024,
      "layer": 0.08966579101979733,
      "mlm": 1.5211062729358673,
      "distill": 1.0546330511569977,
      "teacher_acc": 0.7330964803695679
    },
    {
      "step": 8211,
      "lr": 2.4949653997914978e-05,
      "elapsed_s": 6101.135966539383,
      "loss": 2.891687750816345,
      "hidden": 0.18434017524123192,
      "layer": 0.09039711207151413,
      "mlm": 1.511214554309845,
      "distill": 0.9665941745042801,
      "teacher_acc": 0.719714418053627
    },
    {
      "step": 8221,
      "lr": 2.489930820001099e-05,
      "elapsed_s": 6108.538053035736,
      "loss": 3.153267502784729,
      "hidden": 0.19143395498394966,
      "layer": 0.09137751534581184,
      "mlm": 1.6263537406921387,
      "distill": 1.0983571261167526,
      "teacher_acc": 0.7130441218614578
    },
    {
      "step": 8231,
      "lr": 2.484896281046823e-05,
      "elapsed_s": 6115.918032407761,
      "loss": 2.8901094794273376,
      "hidden": 0.18094927817583084,
      "layer": 0.08909286186099052,
      "mlm": 1.4574635326862335,
      "distill": 1.0262009352445602,
      "teacher_acc": 0.7466393858194351
    },
    {
      "step": 8241,
      "lr": 2.4798618033465258e-05,
      "elapsed_s": 6123.32395863533,
      "loss": 2.931398630142212,
      "hidden": 0.18789293617010117,
      "layer": 0.09053625538945198,
      "mlm": 1.4711494743824005,
      "distill": 1.0391951352357864,
      "teacher_acc": 0.7355734556913376
    },
    {
      "step": 8251,
      "lr": 2.4748274073178114e-05,
      "elapsed_s": 6130.730890274048,
      "loss": 2.9117440581321716,
      "hidden": 0.18057962879538536,
      "layer": 0.08870450221002102,
      "mlm": 1.5162672400474548,
      "distill": 0.9899652898311615,
      "teacher_acc": 0.7124912738800049
    },
    {
      "step": 8261,
      "lr": 2.4697931133779568e-05,
      "elapsed_s": 6138.126479625702,
      "loss": 2.950780153274536,
      "hidden": 0.18413323536515236,
      "layer": 0.09027173183858395,
      "mlm": 1.5216915607452393,
      "distill": 1.015686184167862,
      "teacher_acc": 0.7171885669231415
    },
    {
      "step": 8271,
      "lr": 2.4647589419438224e-05,
      "elapsed_s": 6145.502211809158,
      "loss": 2.9337576627731323,
      "hidden": 0.1831376701593399,
      "layer": 0.08990378491580486,
      "mlm": 1.4892006516456604,
      "distill": 1.0333296656608582,
      "teacher_acc": 0.7304738909006119
    },
    {
      "step": 8281,
      "lr": 2.459724913431772e-05,
      "elapsed_s": 6152.891382694244,
      "loss": 2.939943015575409,
      "hidden": 0.18573177233338356,
      "layer": 0.09129116125404835,
      "mlm": 1.4799895286560059,
      "distill": 1.042844295501709,
      "teacher_acc": 0.7333685755729675
    },
    {
      "step": 8291,
      "lr": 2.4546910482575908e-05,
      "elapsed_s": 6160.266522169113,
      "loss": 3.005593180656433,
      "hidden": 0.18121618777513504,
      "layer": 0.0894862674176693,
      "mlm": 1.5923981964588165,
      "distill": 1.006019487977028,
      "teacher_acc": 0.6983092427253723
    },
    {
      "step": 8301,
      "lr": 2.4496573668363996e-05,
      "elapsed_s": 6167.688009738922,
      "loss": 3.0075316429138184,
      "hidden": 0.18586434051394463,
      "layer": 0.08954703994095325,
      "mlm": 1.5086158514022827,
      "distill": 1.082413598895073,
      "teacher_acc": 0.7297636568546295
    },
    {
      "step": 8311,
      "lr": 2.4446238895825762e-05,
      "elapsed_s": 6175.111860752106,
      "loss": 2.924494981765747,
      "hidden": 0.18256183341145515,
      "layer": 0.0884410459548235,
      "mlm": 1.4452449679374695,
      "distill": 1.069905772805214,
      "teacher_acc": 0.7393586188554764
    },
    {
      "step": 8321,
      "lr": 2.43959063690967e-05,
      "elapsed_s": 6182.510061979294,
      "loss": 2.999220073223114,
      "hidden": 0.1881144382059574,
      "layer": 0.0904888715595007,
      "mlm": 1.5343836843967438,
      "distill": 1.0433630496263504,
      "teacher_acc": 0.7139466404914856
    },
    {
      "step": 8331,
      "lr": 2.4345576292303176e-05,
      "elapsed_s": 6189.89150261879,
      "loss": 2.9658464789390564,
      "hidden": 0.17980826646089554,
      "layer": 0.08801405690610409,
      "mlm": 1.5942479074001312,
      "distill": 0.9679750055074692,
      "teacher_acc": 0.7010392397642136
    },
    {
      "step": 8341,
      "lr": 2.4295248869561644e-05,
      "elapsed_s": 6197.270488500595,
      "loss": 2.9391918182373047,
      "hidden": 0.18330220505595207,
      "layer": 0.08926144987344742,
      "mlm": 1.479045569896698,
      "distill": 1.0489111542701721,
      "teacher_acc": 0.7308953702449799
    },
    {
      "step": 8351,
      "lr": 2.4244924304977785e-05,
      "elapsed_s": 6204.669112920761,
      "loss": 2.875270962715149,
      "hidden": 0.17996390163898468,
      "layer": 0.08799388445913792,
      "mlm": 1.4565063118934631,
      "distill": 1.014840006828308,
      "teacher_acc": 0.7316354215145111
    },
    {
      "step": 8361,
      "lr": 2.4194602802645682e-05,
      "elapsed_s": 6212.076530694962,
      "loss": 2.7997010350227356,
      "hidden": 0.1785064898431301,
      "layer": 0.08745672553777695,
      "mlm": 1.4159866571426392,
      "distill": 0.9829730689525604,
      "teacher_acc": 0.7402416169643402
    },
    {
      "step": 8371,
      "lr": 2.4144284566647015e-05,
      "elapsed_s": 6219.502213001251,
      "loss": 3.0549824237823486,
      "hidden": 0.18566689267754555,
      "layer": 0.09021729417145252,
      "mlm": 1.5863741338253021,
      "distill": 1.052165925502777,
      "teacher_acc": 0.714330643415451
    },
    {
      "step": 8381,
      "lr": 2.40939698010502e-05,
      "elapsed_s": 6226.920089483261,
      "loss": 2.8350305557250977,
      "hidden": 0.18058951944112778,
      "layer": 0.08789621293544769,
      "mlm": 1.4557102918624878,
      "distill": 0.9741931259632111,
      "teacher_acc": 0.7274690866470337
    },
    {
      "step": 8391,
      "lr": 2.4043658709909585e-05,
      "elapsed_s": 6234.333126068115,
      "loss": 2.8359575867652893,
      "hidden": 0.17768486589193344,
      "layer": 0.08745093271136284,
      "mlm": 1.4620692431926727,
      "distill": 0.9747931361198425,
      "teacher_acc": 0.7293361872434616
    },
    {
      "step": 8401,
      "lr": 2.399335149726463e-05,
      "elapsed_s": 6241.73703789711,
      "loss": 2.9529137015342712,
      "hidden": 0.17902612686157227,
      "layer": 0.08773896284401417,
      "mlm": 1.5330709517002106,
      "distill": 1.0179211348295212,
      "teacher_acc": 0.7198192924261093
    },
    {
      "step": 8411,
      "lr": 2.3943048367139034e-05,
      "elapsed_s": 6249.174750089645,
      "loss": 2.8508880138397217,
      "hidden": 0.18088247999548912,
      "layer": 0.08851601183414459,
      "mlm": 1.4675228297710419,
      "distill": 0.9773422181606293,
      "teacher_acc": 0.7270239591598511
    },
    {
      "step": 8421,
      "lr": 2.3892749523539975e-05,
      "elapsed_s": 6256.6026401519775,
      "loss": 2.844675660133362,
      "hidden": 0.18240134790539742,
      "layer": 0.08844905532896519,
      "mlm": 1.4462521076202393,
      "distill": 0.9893963932991028,
      "teacher_acc": 0.7364901006221771
    },
    {
      "step": 8431,
      "lr": 2.3842455170457216e-05,
      "elapsed_s": 6264.001533269882,
      "loss": 2.855037271976471,
      "hidden": 0.17813381552696228,
      "layer": 0.08849231712520123,
      "mlm": 1.465518206357956,
      "distill": 0.9890053272247314,
      "teacher_acc": 0.7295382171869278
    },
    {
      "step": 8441,
      "lr": 2.379216551186233e-05,
      "elapsed_s": 6271.381135225296,
      "loss": 2.854619085788727,
      "hidden": 0.17629743367433548,
      "layer": 0.08741213567554951,
      "mlm": 1.4959725737571716,
      "distill": 0.9623454809188843,
      "teacher_acc": 0.7197598069906235
    },
    {
      "step": 8451,
      "lr": 2.374188075170783e-05,
      "elapsed_s": 6278.794520139694,
      "loss": 2.879940688610077,
      "hidden": 0.18894462287425995,
      "layer": 0.09032072499394417,
      "mlm": 1.4103220999240875,
      "distill": 1.046568974852562,
      "teacher_acc": 0.7382932752370834
    },
    {
      "step": 8461,
      "lr": 2.3691601093926404e-05,
      "elapsed_s": 6286.207994699478,
      "loss": 2.9809134006500244,
      "hidden": 0.18669545277953148,
      "layer": 0.09018196538090706,
      "mlm": 1.5047359466552734,
      "distill": 1.0576955080032349,
      "teacher_acc": 0.7286570072174072
    },
    {
      "step": 8471,
      "lr": 2.3641326742429997e-05,
      "elapsed_s": 6293.6435651779175,
      "loss": 2.907900333404541,
      "hidden": 0.18374108895659447,
      "layer": 0.08859942108392715,
      "mlm": 1.4614209234714508,
      "distill": 1.0346974432468414,
      "teacher_acc": 0.7449843287467957
    },
    {
      "step": 8481,
      "lr": 2.3591057901109063e-05,
      "elapsed_s": 6301.049234628677,
      "loss": 2.9465858340263367,
      "hidden": 0.1828613616526127,
      "layer": 0.08813507668673992,
      "mlm": 1.5369316041469574,
      "distill": 0.9998639822006226,
      "teacher_acc": 0.7159930914640427
    },
    {
      "step": 8491,
      "lr": 2.3540794773831706e-05,
      "elapsed_s": 6308.484765291214,
      "loss": 2.911461055278778,
      "hidden": 0.17863063141703606,
      "layer": 0.08771364018321037,
      "mlm": 1.533032774925232,
      "distill": 0.9773102253675461,
      "teacher_acc": 0.7181064486503601
    },
    {
      "step": 8501,
      "lr": 2.3490537564442847e-05,
      "elapsed_s": 6315.90755033493,
      "loss": 3.124101161956787,
      "hidden": 0.18704332411289215,
      "layer": 0.090591786429286,
      "mlm": 1.6431399285793304,
      "distill": 1.0615786761045456,
      "teacher_acc": 0.6924787163734436
    },
    {
      "step": 8511,
      "lr": 2.344028647676342e-05,
      "elapsed_s": 6323.314769983292,
      "loss": 2.9573127031326294,
      "hidden": 0.1789483092725277,
      "layer": 0.08913538977503777,
      "mlm": 1.5469298958778381,
      "distill": 1.0079184919595718,
      "teacher_acc": 0.7063856422901154
    },
    {
      "step": 8521,
      "lr": 2.3390041714589514e-05,
      "elapsed_s": 6330.7093670368195,
      "loss": 2.9588323831558228,
      "hidden": 0.19193321466445923,
      "layer": 0.09050224162638187,
      "mlm": 1.5065864026546478,
      "distill": 1.023128405213356,
      "teacher_acc": 0.718341127038002
    },
    {
      "step": 8531,
      "lr": 2.333980348169158e-05,
      "elapsed_s": 6338.088020801544,
      "loss": 2.9513808488845825,
      "hidden": 0.1834735907614231,
      "layer": 0.09056422300636768,
      "mlm": 1.4577647745609283,
      "distill": 1.0813868045806885,
      "teacher_acc": 0.7399847656488419
    },
    {
      "step": 8541,
      "lr": 2.3289571981813594e-05,
      "elapsed_s": 6345.518124341965,
      "loss": 3.0267804861068726,
      "hidden": 0.1823589764535427,
      "layer": 0.08937833458185196,
      "mlm": 1.5498903095722198,
      "distill": 1.067482978105545,
      "teacher_acc": 0.7257059216499329
    },
    {
      "step": 8551,
      "lr": 2.3239347418672202e-05,
      "elapsed_s": 6352.943686723709,
      "loss": 2.8283453583717346,
      "hidden": 0.17376576364040375,
      "layer": 0.08760103024542332,
      "mlm": 1.4394747912883759,
      "distill": 0.9975385218858719,
      "teacher_acc": 0.7404749691486359
    },
    {
      "step": 8561,
      "lr": 2.3189129995955943e-05,
      "elapsed_s": 6360.37038564682,
      "loss": 2.891727030277252,
      "hidden": 0.18154527619481087,
      "layer": 0.08857088536024094,
      "mlm": 1.493547797203064,
      "distill": 0.9908032566308975,
      "teacher_acc": 0.7230571210384369
    },
    {
      "step": 8571,
      "lr": 2.3138919917324378e-05,
      "elapsed_s": 6367.7848508358,
      "loss": 2.8875765800476074,
      "hidden": 0.17877573892474174,
      "layer": 0.08766302280128002,
      "mlm": 1.46577787399292,
      "distill": 1.0204157084226608,
      "teacher_acc": 0.7307460159063339
    },
    {
      "step": 8581,
      "lr": 2.3088717386407297e-05,
      "elapsed_s": 6375.205098867416,
      "loss": 3.0409688353538513,
      "hidden": 0.19034617394208908,
      "layer": 0.09154427237808704,
      "mlm": 1.5139701664447784,
      "distill": 1.100534200668335,
      "teacher_acc": 0.7252182960510254
    },
    {
      "step": 8591,
      "lr": 2.303852260680388e-05,
      "elapsed_s": 6382.615610122681,
      "loss": 2.9315292835235596,
      "hidden": 0.18493446335196495,
      "layer": 0.08950814232230186,
      "mlm": 1.5043656527996063,
      "distill": 1.0125406682491302,
      "teacher_acc": 0.7180290371179581
    },
    {
      "step": 8601,
      "lr": 2.2988335782081855e-05,
      "elapsed_s": 6390.003125667572,
      "loss": 2.895297586917877,
      "hidden": 0.183767668902874,
      "layer": 0.09037392027676105,
      "mlm": 1.468329668045044,
      "distill": 1.0142456740140915,
      "teacher_acc": 0.7316508889198303
    },
    {
      "step": 8611,
      "lr": 2.2938157115776704e-05,
      "elapsed_s": 6397.380096912384,
      "loss": 2.93588924407959,
      "hidden": 0.18244516849517822,
      "layer": 0.0886326152831316,
      "mlm": 1.4861495792865753,
      "distill": 1.0405329763889313,
      "teacher_acc": 0.7311215102672577
    },
    {
      "step": 8621,
      "lr": 2.2887986811390828e-05,
      "elapsed_s": 6404.787356138229,
      "loss": 2.8331461548805237,
      "hidden": 0.1785254329442978,
      "layer": 0.08767159096896648,
      "mlm": 1.443150371313095,
      "distill": 0.9891091287136078,
      "teacher_acc": 0.7302334606647491
    },
    {
      "step": 8631,
      "lr": 2.283782507239269e-05,
      "elapsed_s": 6412.200192451477,
      "loss": 2.934127449989319,
      "hidden": 0.18885766714811325,
      "layer": 0.09050243906676769,
      "mlm": 1.477938860654831,
      "distill": 1.0332220792770386,
      "teacher_acc": 0.7288858443498611
    },
    {
      "step": 8641,
      "lr": 2.2787672102216042e-05,
      "elapsed_s": 6419.626239776611,
      "loss": 2.8575106859207153,
      "hidden": 0.17604610696434975,
      "layer": 0.0872959028929472,
      "mlm": 1.5211619436740875,
      "distill": 0.9406086206436157,
      "teacher_acc": 0.7169828861951828
    },
    {
      "step": 8651,
      "lr": 2.2737528104259056e-05,
      "elapsed_s": 6427.039954900742,
      "loss": 3.001944899559021,
      "hidden": 0.1855383887887001,
      "layer": 0.09014837816357613,
      "mlm": 1.5173896551132202,
      "distill": 1.0684043169021606,
      "teacher_acc": 0.7207956612110138
    },
    {
      "step": 8661,
      "lr": 2.2687393281883525e-05,
      "elapsed_s": 6434.469883680344,
      "loss": 2.897132694721222,
      "hidden": 0.1799173578619957,
      "layer": 0.08801806345582008,
      "mlm": 1.493676781654358,
      "distill": 0.9996120929718018,
      "teacher_acc": 0.7295356690883636
    },
    {
      "step": 8671,
      "lr": 2.263726783841404e-05,
      "elapsed_s": 6441.8876984119415,
      "loss": 3.007726728916168,
      "hidden": 0.18790575116872787,
      "layer": 0.08982766047120094,
      "mlm": 1.576948344707489,
      "distill": 1.0100530236959457,
      "teacher_acc": 0.7038985043764114
    },
    {
      "step": 8681,
      "lr": 2.2587151977137122e-05,
      "elapsed_s": 6449.317623376846,
      "loss": 2.8331122994422913,
      "hidden": 0.17551447078585625,
      "layer": 0.08662776090204716,
      "mlm": 1.465342491865158,
      "distill": 0.9734269827604294,
      "teacher_acc": 0.730484127998352
    },
    {
      "step": 8691,
      "lr": 2.2537045901300476e-05,
      "elapsed_s": 6456.731172084808,
      "loss": 2.968348264694214,
      "hidden": 0.18644027784466743,
      "layer": 0.08992563746869564,
      "mlm": 1.548774242401123,
      "distill": 1.0017306208610535,
      "teacher_acc": 0.7173354029655457
    },
    {
      "step": 8701,
      "lr": 2.2486949814112077e-05,
      "elapsed_s": 6464.127074480057,
      "loss": 2.807942032814026,
      "hidden": 0.18399015441536903,
      "layer": 0.08863610774278641,
      "mlm": 1.4037708938121796,
      "distill": 0.9918727874755859,
      "teacher_acc": 0.7371050417423248
    },
    {
      "step": 8711,
      "lr": 2.243686391873942e-05,
      "elapsed_s": 6471.529432058334,
      "loss": 2.965940296649933,
      "hidden": 0.178830087184906,
      "layer": 0.08769601956009865,
      "mlm": 1.5386537313461304,
      "distill": 1.025778368115425,
      "teacher_acc": 0.7213774770498276
    },
    {
      "step": 8721,
      "lr": 2.238678841830867e-05,
      "elapsed_s": 6478.931056022644,
      "loss": 2.9844260811805725,
      "hidden": 0.1846175603568554,
      "layer": 0.08937561511993408,
      "mlm": 1.5243127644062042,
      "distill": 1.0461903810501099,
      "teacher_acc": 0.7168848812580109
    },
    {
      "step": 8731,
      "lr": 2.2336723515903803e-05,
      "elapsed_s": 6486.3247056007385,
      "loss": 3.0020660161972046,
      "hidden": 0.18470419198274612,
      "layer": 0.09019277431070805,
      "mlm": 1.5129064917564392,
      "distill": 1.0746547877788544,
      "teacher_acc": 0.7282703816890717
    },
    {
      "step": 8741,
      "lr": 2.2286669414565846e-05,
      "elapsed_s": 6493.715079069138,
      "loss": 2.9585015773773193,
      "hidden": 0.18602298945188522,
      "layer": 0.09012051299214363,
      "mlm": 1.5135542750358582,
      "distill": 1.0278410464525223,
      "teacher_acc": 0.7184338569641113
    },
    {
      "step": 8751,
      "lr": 2.2236626317292007e-05,
      "elapsed_s": 6501.108938217163,
      "loss": 2.892183244228363,
      "hidden": 0.18466924875974655,
      "layer": 0.08989451080560684,
      "mlm": 1.4613795280456543,
      "distill": 1.0165180265903473,
      "teacher_acc": 0.7374773770570755
    },
    {
      "step": 8761,
      "lr": 2.2186594427034864e-05,
      "elapsed_s": 6508.503279685974,
      "loss": 2.7361000776290894,
      "hidden": 0.18135832995176315,
      "layer": 0.08705575577914715,
      "mlm": 1.370358556509018,
      "distill": 0.9594970494508743,
      "teacher_acc": 0.749190092086792
    },
    {
      "step": 8771,
      "lr": 2.213657394670156e-05,
      "elapsed_s": 6515.889404535294,
      "loss": 3.054608464241028,
      "hidden": 0.186349805444479,
      "layer": 0.08988721296191216,
      "mlm": 1.5879962146282196,
      "distill": 1.0489690899848938,
      "teacher_acc": 0.707442045211792
    },
    {
      "step": 8781,
      "lr": 2.2086565079152946e-05,
      "elapsed_s": 6523.287684440613,
      "loss": 3.07598340511322,
      "hidden": 0.18410733714699745,
      "layer": 0.08933237940073013,
      "mlm": 1.6352986693382263,
      "distill": 1.0278037935495377,
      "teacher_acc": 0.6948581784963608
    },
    {
      "step": 8791,
      "lr": 2.2036568027202793e-05,
      "elapsed_s": 6530.70818901062,
      "loss": 2.8574676513671875,
      "hidden": 0.18139314278960228,
      "layer": 0.08717617951333523,
      "mlm": 1.412856012582779,
      "distill": 1.0382372438907623,
      "teacher_acc": 0.7452773004770279
    },
    {
      "step": 8801,
      "lr": 2.1986582993616926e-05,
      "elapsed_s": 6538.117563724518,
      "loss": 3.0106810331344604,
      "hidden": 0.186729047447443,
      "layer": 0.08938080817461014,
      "mlm": 1.5481558740139008,
      "distill": 1.0443766415119171,
      "teacher_acc": 0.7178986072540283
    },
    {
      "step": 8811,
      "lr": 2.1936610181112466e-05,
      "elapsed_s": 6545.558225154877,
      "loss": 2.9059844613075256,
      "hidden": 0.18045691773295403,
      "layer": 0.08874400332570076,
      "mlm": 1.4922977983951569,
      "distill": 1.008400782942772,
      "teacher_acc": 0.7315785884857178
    },
    {
      "step": 8821,
      "lr": 2.188664979235695e-05,
      "elapsed_s": 6552.98779630661,
      "loss": 2.933813989162445,
      "hidden": 0.17986387014389038,
      "layer": 0.08861134760081768,
      "mlm": 1.5210449397563934,
      "distill": 1.008735716342926,
      "teacher_acc": 0.7107240557670593
    },
    {
      "step": 8831,
      "lr": 2.1836702029967524e-05,
      "elapsed_s": 6560.388173818588,
      "loss": 2.98126620054245,
      "hidden": 0.1842743642628193,
      "layer": 0.08878319524228573,
      "mlm": 1.5688044130802155,
      "distill": 0.9995214939117432,
      "teacher_acc": 0.7114357650279999
    },
    {
      "step": 8841,
      "lr": 2.1786767096510137e-05,
      "elapsed_s": 6567.799901485443,
      "loss": 3.0354889035224915,
      "hidden": 0.18559569492936134,
      "layer": 0.08919881843030453,
      "mlm": 1.59492626786232,
      "distill": 1.0247717499732971,
      "teacher_acc": 0.7039584666490555
    },
    {
      "step": 8851,
      "lr": 2.173684519449872e-05,
      "elapsed_s": 6575.241186618805,
      "loss": 2.9517104029655457,
      "hidden": 0.1806754171848297,
      "layer": 0.08847871236503124,
      "mlm": 1.5132551491260529,
      "distill": 1.0328650325536728,
      "teacher_acc": 0.7282463759183884
    },
    {
      "step": 8861,
      "lr": 2.168693652639432e-05,
      "elapsed_s": 6582.639365911484,
      "loss": 2.963459610939026,
      "hidden": 0.18071004375815392,
      "layer": 0.08869470097124577,
      "mlm": 1.529125064611435,
      "distill": 1.028567060828209,
      "teacher_acc": 0.7226865887641907
    },
    {
      "step": 8871,
      "lr": 2.1637041294604353e-05,
      "elapsed_s": 6590.060245037079,
      "loss": 2.9387542009353638,
      "hidden": 0.1904049515724182,
      "layer": 0.08992573246359825,
      "mlm": 1.4812887609004974,
      "distill": 1.0316926687955856,
      "teacher_acc": 0.7294967919588089
    },
    {
      "step": 8881,
      "lr": 2.1587159701481716e-05,
      "elapsed_s": 6597.451440572739,
      "loss": 3.0799967646598816,
      "hidden": 0.18595696613192558,
      "layer": 0.08928271569311619,
      "mlm": 1.5693759322166443,
      "distill": 1.0940655171871185,
      "teacher_acc": 0.7186525762081146
    },
    {
      "step": 8891,
      "lr": 2.1537291949324007e-05,
      "elapsed_s": 6604.86493396759,
      "loss": 2.8588709831237793,
      "hidden": 0.1773664988577366,
      "layer": 0.08765405975282192,
      "mlm": 1.4873344600200653,
      "distill": 0.9729765802621841,
      "teacher_acc": 0.7125709652900696
    },
    {
      "step": 8901,
      "lr": 2.148743824037269e-05,
      "elapsed_s": 6612.294847488403,
      "loss": 2.8426312804222107,
      "hidden": 0.17505457997322083,
      "layer": 0.08710480853915215,
      "mlm": 1.4908365607261658,
      "distill": 0.9581330865621567,
      "teacher_acc": 0.7272351384162903
    },
    {
      "step": 8911,
      "lr": 2.1437598776812263e-05,
      "elapsed_s": 6619.741770267487,
      "loss": 2.913152039051056,
      "hidden": 0.1833137758076191,
      "layer": 0.08867250569164753,
      "mlm": 1.4868301451206207,
      "distill": 1.015358105301857,
      "teacher_acc": 0.7328678071498871
    },
    {
      "step": 8921,
      "lr": 2.1387773760769474e-05,
      "elapsed_s": 6627.171642541885,
      "loss": 2.8973346948623657,
      "hidden": 0.18192962184548378,
      "layer": 0.08837432600557804,
      "mlm": 1.49186971783638,
      "distill": 0.9974185973405838,
      "teacher_acc": 0.7323737889528275
    },
    {
      "step": 8931,
      "lr": 2.1337963394312455e-05,
      "elapsed_s": 6634.618430137634,
      "loss": 2.9903522729873657,
      "hidden": 0.17788656428456306,
      "layer": 0.08825463242828846,
      "mlm": 1.555762529373169,
      "distill": 1.0346892476081848,
      "teacher_acc": 0.7117421180009842
    },
    {
      "step": 8941,
      "lr": 2.1288167879449935e-05,
      "elapsed_s": 6642.035146951675,
      "loss": 2.7845088243484497,
      "hidden": 0.18205909058451653,
      "layer": 0.08897701278328896,
      "mlm": 1.3837000131607056,
      "distill": 0.9922020733356476,
      "teacher_acc": 0.7477200031280518
    },
    {
      "step": 8951,
      "lr": 2.1238387418130422e-05,
      "elapsed_s": 6649.45331454277,
      "loss": 2.9792811274528503,
      "hidden": 0.18322540819644928,
      "layer": 0.08939226157963276,
      "mlm": 1.54749995470047,
      "distill": 1.0206342041492462,
      "teacher_acc": 0.7230598330497742
    },
    {
      "step": 8961,
      "lr": 2.1188622212241367e-05,
      "elapsed_s": 6656.87856388092,
      "loss": 3.056369185447693,
      "hidden": 0.18552593886852264,
      "layer": 0.09026428684592247,
      "mlm": 1.5303844511508942,
      "distill": 1.1098006963729858,
      "teacher_acc": 0.7350148558616638
    },
    {
      "step": 8971,
      "lr": 2.1138872463608334e-05,
      "elapsed_s": 6664.315057277679,
      "loss": 3.1146926283836365,
      "hidden": 0.1871957816183567,
      "layer": 0.09087642095983028,
      "mlm": 1.6493154466152191,
      "distill": 1.0455473810434341,
      "teacher_acc": 0.7042504251003265
    },
    {
      "step": 8981,
      "lr": 2.1089138373994223e-05,
      "elapsed_s": 6671.74267911911,
      "loss": 2.969256639480591,
      "hidden": 0.1860981360077858,
      "layer": 0.09092613309621811,
      "mlm": 1.4887465834617615,
      "distill": 1.0628507435321808,
      "teacher_acc": 0.7439458817243576
    },
    {
      "step": 8991,
      "lr": 2.1039420145098426e-05,
      "elapsed_s": 6679.1408042907715,
      "loss": 3.039336919784546,
      "hidden": 0.1812228485941887,
      "layer": 0.08929944410920143,
      "mlm": 1.5841592252254486,
      "distill": 1.0480823814868927,
      "teacher_acc": 0.7157245576381683
    },
    {
      "step": 9001,
      "lr": 2.098971797855599e-05,
      "elapsed_s": 6688.556682348251,
      "loss": 2.8443188667297363,
      "hidden": 0.17946211621165276,
      "layer": 0.08726046979427338,
      "mlm": 1.4529588520526886,
      "distill": 0.9888056218624115,
      "teacher_acc": 0.7396171540021896
    },
    {
      "step": 9011,
      "lr": 2.094003207593685e-05,
      "elapsed_s": 6695.9745762348175,
      "loss": 2.9383957386016846,
      "hidden": 0.1811458021402359,
      "layer": 0.08890057168900967,
      "mlm": 1.5210141837596893,
      "distill": 1.010639637708664,
      "teacher_acc": 0.728861078619957
    },
    {
      "step": 9021,
      "lr": 2.0890362638744952e-05,
      "elapsed_s": 6703.384733438492,
      "loss": 3.0423710346221924,
      "hidden": 0.18503785505890846,
      "layer": 0.08943272940814495,
      "mlm": 1.566678375005722,
      "distill": 1.0609004944562912,
      "teacher_acc": 0.7249512076377869
    },
    {
      "step": 9031,
      "lr": 2.084070986841748e-05,
      "elapsed_s": 6710.791173458099,
      "loss": 3.006297469139099,
      "hidden": 0.18876268342137337,
      "layer": 0.09146064519882202,
      "mlm": 1.5095337331295013,
      "distill": 1.0735080242156982,
      "teacher_acc": 0.7247065156698227
    },
    {
      "step": 9041,
      "lr": 2.0791073966324037e-05,
      "elapsed_s": 6718.199744224548,
      "loss": 2.9743499755859375,
      "hidden": 0.18308956176042557,
      "layer": 0.08879238180816174,
      "mlm": 1.5403054058551788,
      "distill": 1.0234692841768265,
      "teacher_acc": 0.7184507697820663
    },
    {
      "step": 9051,
      "lr": 2.0741455133765792e-05,
      "elapsed_s": 6725.576699733734,
      "loss": 2.9561312794685364,
      "hidden": 0.18552611023187637,
      "layer": 0.08953426405787468,
      "mlm": 1.487838089466095,
      "distill": 1.0524738132953644,
      "teacher_acc": 0.7321216762065887
    },
    {
      "step": 9061,
      "lr": 2.0691853571974707e-05,
      "elapsed_s": 6732.906847715378,
      "loss": 2.9539469480514526,
      "hidden": 0.1784037910401821,
      "layer": 0.08854863233864307,
      "mlm": 1.5525899231433868,
      "distill": 1.0002751350402832,
      "teacher_acc": 0.712956041097641
    },
    {
      "step": 9071,
      "lr": 2.064226948211268e-05,
      "elapsed_s": 6740.290674448013,
      "loss": 2.8657299876213074,
      "hidden": 0.17860187962651253,
      "layer": 0.08830633945763111,
      "mlm": 1.4841061234474182,
      "distill": 0.9802669137716293,
      "teacher_acc": 0.7153952717781067
    },
    {
      "step": 9081,
      "lr": 2.059270306527078e-05,
      "elapsed_s": 6747.6943900585175,
      "loss": 2.925731837749481,
      "hidden": 0.18265193328261375,
      "layer": 0.08962615951895714,
      "mlm": 1.4300443530082703,
      "distill": 1.085570588707924,
      "teacher_acc": 0.7369037866592407
    },
    {
      "step": 9091,
      "lr": 2.0543154522468378e-05,
      "elapsed_s": 6755.078178167343,
      "loss": 2.9933385848999023,
      "hidden": 0.18611905723810196,
      "layer": 0.08938478119671345,
      "mlm": 1.5744709074497223,
      "distill": 1.0019371956586838,
      "teacher_acc": 0.6952493041753769
    },
    {
      "step": 9101,
      "lr": 2.0493624054652357e-05,
      "elapsed_s": 6762.4684002399445,
      "loss": 2.8948601484298706,
      "hidden": 0.17796407639980316,
      "layer": 0.08869042247533798,
      "mlm": 1.5196565091609955,
      "distill": 0.9749303013086319,
      "teacher_acc": 0.7197262048721313
    },
    {
      "step": 9111,
      "lr": 2.0444111862696314e-05,
      "elapsed_s": 6769.887577056885,
      "loss": 3.0102808475494385,
      "hidden": 0.18353719636797905,
      "layer": 0.08963865600526333,
      "mlm": 1.59958216547966,
      "distill": 0.9988049864768982,
      "teacher_acc": 0.6965661942958832
    },
    {
      "step": 9121,
      "lr": 2.0394618147399713e-05,
      "elapsed_s": 6777.316593170166,
      "loss": 2.9312583208084106,
      "hidden": 0.17946429178118706,
      "layer": 0.0885950792580843,
      "mlm": 1.515754371881485,
      "distill": 1.0122778415679932,
      "teacher_acc": 0.7262444049119949
    },
    {
      "step": 9131,
      "lr": 2.0345143109487083e-05,
      "elapsed_s": 6784.73273396492,
      "loss": 3.0623274445533752,
      "hidden": 0.183815635740757,
      "layer": 0.08891385607421398,
      "mlm": 1.572904258966446,
      "distill": 1.0773349553346634,
      "teacher_acc": 0.7191461026668549
    },
    {
      "step": 9141,
      "lr": 2.029568694960722e-05,
      "elapsed_s": 6792.169924259186,
      "loss": 2.833001971244812,
      "hidden": 0.18117699772119522,
      "layer": 0.08910947479307652,
      "mlm": 1.3731937408447266,
      "distill": 1.0528995096683502,
      "teacher_acc": 0.7619260400533676
    },
    {
      "step": 9151,
      "lr": 2.0246249868332336e-05,
      "elapsed_s": 6799.594704151154,
      "loss": 3.013506054878235,
      "hidden": 0.1850515715777874,
      "layer": 0.0883651040494442,
      "mlm": 1.5346341133117676,
      "distill": 1.0645862519741058,
      "teacher_acc": 0.7193938791751862
    },
    {
      "step": 9161,
      "lr": 2.019683206615729e-05,
      "elapsed_s": 6807.020638227463,
      "loss": 2.927738904953003,
      "hidden": 0.18236613646149635,
      "layer": 0.08965080976486206,
      "mlm": 1.474910020828247,
      "distill": 1.043271318078041,
      "teacher_acc": 0.7396684736013412
    },
    {
      "step": 9171,
      "lr": 2.014743374349875e-05,
      "elapsed_s": 6814.445960521698,
      "loss": 2.9673004150390625,
      "hidden": 0.18666526675224304,
      "layer": 0.08934594504535198,
      "mlm": 1.465211659669876,
      "distill": 1.0840852558612823,
      "teacher_acc": 0.7396788746118546
    },
    {
      "step": 9181,
      "lr": 2.0098055100694376e-05,
      "elapsed_s": 6821.8679003715515,
      "loss": 2.9574989676475525,
      "hidden": 0.18415352329611778,
      "layer": 0.08890662156045437,
      "mlm": 1.5070875883102417,
      "distill": 1.0376510918140411,
      "teacher_acc": 0.7322539985179901
    },
    {
      "step": 9191,
      "lr": 2.0048696338002015e-05,
      "elapsed_s": 6829.272068977356,
      "loss": 2.969307065010071,
      "hidden": 0.18166471272706985,
      "layer": 0.0888749361038208,
      "mlm": 1.5413909256458282,
      "distill": 1.0201491713523865,
      "teacher_acc": 0.7219777256250381
    },
    {
      "step": 9201,
      "lr": 1.9999357655598893e-05,
      "elapsed_s": 6836.698894023895,
      "loss": 2.9928473830223083,
      "hidden": 0.1843622401356697,
      "layer": 0.08878703974187374,
      "mlm": 1.5087529718875885,
      "distill": 1.0709763765335083,
      "teacher_acc": 0.728574812412262
    },
    {
      "step": 9211,
      "lr": 1.99500392535808e-05,
      "elapsed_s": 6844.135104417801,
      "loss": 2.925539970397949,
      "hidden": 0.18351759016513824,
      "layer": 0.08963667042553425,
      "mlm": 1.5214453339576721,
      "distill": 0.9922410994768143,
      "teacher_acc": 0.7187430411577225
    },
    {
      "step": 9221,
      "lr": 1.990074133196129e-05,
      "elapsed_s": 6851.542471885681,
      "loss": 2.9569990634918213,
      "hidden": 0.1818954087793827,
      "layer": 0.08974380418658257,
      "mlm": 1.5206761956214905,
      "distill": 1.0276601314544678,
      "teacher_acc": 0.7194148004055023
    },
    {
      "step": 9231,
      "lr": 1.9851464090670817e-05,
      "elapsed_s": 6858.9192225933075,
      "loss": 2.967846930027008,
      "hidden": 0.1782795526087284,
      "layer": 0.08730392158031464,
      "mlm": 1.5537519454956055,
      "distill": 1.013883963227272,
      "teacher_acc": 0.7156500369310379
    },
    {
      "step": 9241,
      "lr": 1.980220772955602e-05,
      "elapsed_s": 6866.332326889038,
      "loss": 2.9644923210144043,
      "hidden": 0.18977142870426178,
      "layer": 0.09126338362693787,
      "mlm": 1.4899572432041168,
      "distill": 1.0493605732917786,
      "teacher_acc": 0.7206709831953049
    },
    {
      "step": 9251,
      "lr": 1.9752972448378814e-05,
      "elapsed_s": 6873.742569684982,
      "loss": 2.919346332550049,
      "hidden": 0.1817329190671444,
      "layer": 0.08774051256477833,
      "mlm": 1.5148523151874542,
      "distill": 0.9971580058336258,
      "teacher_acc": 0.713471457362175
    },
    {
      "step": 9261,
      "lr": 1.9703758446815644e-05,
      "elapsed_s": 6881.163727760315,
      "loss": 2.9146288633346558,
      "hidden": 0.1810181438922882,
      "layer": 0.08793342486023903,
      "mlm": 1.5002846121788025,
      "distill": 1.00834122300148,
      "teacher_acc": 0.7204383313655853
    },
    {
      "step": 9271,
      "lr": 1.965456592445665e-05,
      "elapsed_s": 6888.547842264175,
      "loss": 2.9853222966194153,
      "hidden": 0.1813402846455574,
      "layer": 0.08915203623473644,
      "mlm": 1.5814407765865326,
      "distill": 0.9966249614953995,
      "teacher_acc": 0.7130928039550781
    },
    {
      "step": 9281,
      "lr": 1.960539508080485e-05,
      "elapsed_s": 6895.972282171249,
      "loss": 3.0880581736564636,
      "hidden": 0.1843983605504036,
      "layer": 0.09014306217432022,
      "mlm": 1.5965635478496552,
      "distill": 1.0776262879371643,
      "teacher_acc": 0.7143307626247406
    },
    {
      "step": 9291,
      "lr": 1.9556246115275363e-05,
      "elapsed_s": 6903.398527622223,
      "loss": 2.885476231575012,
      "hidden": 0.18250679969787598,
      "layer": 0.08813094906508923,
      "mlm": 1.4706672728061676,
      "distill": 1.0057299733161926,
      "teacher_acc": 0.7248648554086685
    },
    {
      "step": 9301,
      "lr": 1.950711922719458e-05,
      "elapsed_s": 6910.838722705841,
      "loss": 3.0008015036582947,
      "hidden": 0.1779869943857193,
      "layer": 0.0865168645977974,
      "mlm": 1.596402257680893,
      "distill": 1.0051668286323547,
      "teacher_acc": 0.7009795606136322
    },
    {
      "step": 9311,
      "lr": 1.9458014615799326e-05,
      "elapsed_s": 6918.265877962112,
      "loss": 3.0652342438697815,
      "hidden": 0.18569746986031532,
      "layer": 0.08882743120193481,
      "mlm": 1.6116750538349152,
      "distill": 1.0377505719661713,
      "teacher_acc": 0.705446183681488
    },
    {
      "step": 9321,
      "lr": 1.9408932480236118e-05,
      "elapsed_s": 6925.684828281403,
      "loss": 2.8885658383369446,
      "hidden": 0.1742679961025715,
      "layer": 0.08710605278611183,
      "mlm": 1.5368615984916687,
      "distill": 0.9596152454614639,
      "teacher_acc": 0.7066280841827393
    },
    {
      "step": 9331,
      "lr": 1.9359873019560298e-05,
      "elapsed_s": 6933.095467805862,
      "loss": 2.8027408123016357,
      "hidden": 0.17293189838528633,
      "layer": 0.08650610409677029,
      "mlm": 1.4783698618412018,
      "distill": 0.9352541267871857,
      "teacher_acc": 0.7269388437271118
    },
    {
      "step": 9341,
      "lr": 1.931083643273525e-05,
      "elapsed_s": 6940.491014957428,
      "loss": 3.0786247849464417,
      "hidden": 0.189407829195261,
      "layer": 0.09113780222833157,
      "mlm": 1.524317055940628,
      "distill": 1.1299230754375458,
      "teacher_acc": 0.7357308566570282
    },
    {
      "step": 9351,
      "lr": 1.926182291863162e-05,
      "elapsed_s": 6947.870617866516,
      "loss": 2.8821210861206055,
      "hidden": 0.18381726369261742,
      "layer": 0.08906432054936886,
      "mlm": 1.4629663527011871,
      "distill": 1.0069881081581116,
      "teacher_acc": 0.7326804995536804
    },
    {
      "step": 9361,
      "lr": 1.921283267602643e-05,
      "elapsed_s": 6955.265247106552,
      "loss": 3.0665568709373474,
      "hidden": 0.18284516409039497,
      "layer": 0.08873661793768406,
      "mlm": 1.607182115316391,
      "distill": 1.0493161529302597,
      "teacher_acc": 0.7015600055456161
    },
    {
      "step": 9371,
      "lr": 1.9163865903602374e-05,
      "elapsed_s": 6962.685258150101,
      "loss": 2.99079567193985,
      "hidden": 0.18241433799266815,
      "layer": 0.08970578387379646,
      "mlm": 1.5495185554027557,
      "distill": 1.0315955579280853,
      "teacher_acc": 0.7178710252046585
    },
    {
      "step": 9381,
      "lr": 1.9114922799946923e-05,
      "elapsed_s": 6970.088163137436,
      "loss": 2.9857935905456543,
      "hidden": 0.18073765188455582,
      "layer": 0.08737485855817795,
      "mlm": 1.5455012321472168,
      "distill": 1.0351296961307526,
      "teacher_acc": 0.7217265367507935
    },
    {
      "step": 9391,
      "lr": 1.9066003563551584e-05,
      "elapsed_s": 6977.464441776276,
      "loss": 2.9821006655693054,
      "hidden": 0.18190068379044533,
      "layer": 0.08923388831317425,
      "mlm": 1.5737967789173126,
      "distill": 0.999885618686676,
      "teacher_acc": 0.7039564102888107
    },
    {
      "step": 9401,
      "lr": 1.9017108392811065e-05,
      "elapsed_s": 6984.848764896393,
      "loss": 2.9337278008461,
      "hidden": 0.18196579068899155,
      "layer": 0.08965999446809292,
      "mlm": 1.5058395564556122,
      "distill": 1.0191267430782318,
      "teacher_acc": 0.7280523777008057
    },
    {
      "step": 9411,
      "lr": 1.8968237486022454e-05,
      "elapsed_s": 6992.211396694183,
      "loss": 2.858056604862213,
      "hidden": 0.17986347898840904,
      "layer": 0.0875780712813139,
      "mlm": 1.4409327507019043,
      "distill": 1.0136079639196396,
      "teacher_acc": 0.7363061457872391
    },
    {
      "step": 9421,
      "lr": 1.891939104138445e-05,
      "elapsed_s": 6999.572383880615,
      "loss": 2.89581698179245,
      "hidden": 0.17713701725006104,
      "layer": 0.08852677047252655,
      "mlm": 1.5470444858074188,
      "distill": 0.9502349942922592,
      "teacher_acc": 0.7144889384508133
    },
    {
      "step": 9431,
      "lr": 1.887056925699656e-05,
      "elapsed_s": 7006.954679965973,
      "loss": 2.7948920726776123,
      "hidden": 0.17615030705928802,
      "layer": 0.08674420043826103,
      "mlm": 1.4227837920188904,
      "distill": 0.9764355272054672,
      "teacher_acc": 0.7339392006397247
    },
    {
      "step": 9441,
      "lr": 1.882177233085826e-05,
      "elapsed_s": 7014.334565401077,
      "loss": 3.002918004989624,
      "hidden": 0.18485255166888237,
      "layer": 0.0895133875310421,
      "mlm": 1.5169658362865448,
      "distill": 1.071490302681923,
      "teacher_acc": 0.7285916954278946
    },
    {
      "step": 9451,
      "lr": 1.8773000460868205e-05,
      "elapsed_s": 7021.76203250885,
      "loss": 2.847712218761444,
      "hidden": 0.18119439482688904,
      "layer": 0.08906062878668308,
      "mlm": 1.4232844412326813,
      "distill": 1.0175086855888367,
      "teacher_acc": 0.7444396317005157
    },
    {
      "step": 9461,
      "lr": 1.872425384482346e-05,
      "elapsed_s": 7029.186311721802,
      "loss": 2.874585211277008,
      "hidden": 0.18441222980618477,
      "layer": 0.088089969009161,
      "mlm": 1.4695186614990234,
      "distill": 0.9921971559524536,
      "teacher_acc": 0.7358139455318451
    },
    {
      "step": 9471,
      "lr": 1.867553268041864e-05,
      "elapsed_s": 7036.588424682617,
      "loss": 2.9314531683921814,
      "hidden": 0.18103083223104477,
      "layer": 0.08737107552587986,
      "mlm": 1.5569888055324554,
      "distill": 0.9687171280384064,
      "teacher_acc": 0.7083883136510849
    },
    {
      "step": 9481,
      "lr": 1.8626837165245165e-05,
      "elapsed_s": 7044.007306575775,
      "loss": 2.87921142578125,
      "hidden": 0.18087473139166832,
      "layer": 0.08881294913589954,
      "mlm": 1.489160418510437,
      "distill": 0.9838951081037521,
      "teacher_acc": 0.7239366918802261
    },
    {
      "step": 9491,
      "lr": 1.857816749679043e-05,
      "elapsed_s": 7051.432197809219,
      "loss": 2.987193465232849,
      "hidden": 0.18151448667049408,
      "layer": 0.08880833722651005,
      "mlm": 1.546518087387085,
      "distill": 1.0332422107458115,
      "teacher_acc": 0.7137254029512405
    },
    {
      "step": 9501,
      "lr": 1.852952387243698e-05,
      "elapsed_s": 7058.868223905563,
      "loss": 3.048856198787689,
      "hidden": 0.1871863380074501,
      "layer": 0.08909785002470016,
      "mlm": 1.5878590047359467,
      "distill": 1.0420755296945572,
      "teacher_acc": 0.7062763124704361
    },
    {
      "step": 9511,
      "lr": 1.8480906489461777e-05,
      "elapsed_s": 7066.283197641373,
      "loss": 2.8597522974014282,
      "hidden": 0.17677029967308044,
      "layer": 0.08772431313991547,
      "mlm": 1.477822870016098,
      "distill": 0.9845266491174698,
      "teacher_acc": 0.7231673300266266
    },
    {
      "step": 9521,
      "lr": 1.8432315545035328e-05,
      "elapsed_s": 7073.712001085281,
      "loss": 2.9775474667549133,
      "hidden": 0.1737140603363514,
      "layer": 0.0866665430366993,
      "mlm": 1.5912586450576782,
      "distill": 0.9955274164676666,
      "teacher_acc": 0.7076480388641357
    },
    {
      "step": 9531,
      "lr": 1.8383751236220932e-05,
      "elapsed_s": 7081.1467208862305,
      "loss": 2.914586842060089,
      "hidden": 0.17986729368567467,
      "layer": 0.08838069252669811,
      "mlm": 1.4901664853096008,
      "distill": 1.020495444536209,
      "teacher_acc": 0.7296864837408066
    },
    {
      "step": 9541,
      "lr": 1.8335213759973867e-05,
      "elapsed_s": 7088.537719964981,
      "loss": 2.771449565887451,
      "hidden": 0.17355427145957947,
      "layer": 0.08566816709935665,
      "mlm": 1.4420372545719147,
      "distill": 0.939469650387764,
      "teacher_acc": 0.7277169972658157
    },
    {
      "step": 9551,
      "lr": 1.828670331314058e-05,
      "elapsed_s": 7095.962240695953,
      "loss": 2.972373068332672,
      "hidden": 0.18390975520014763,
      "layer": 0.088649220764637,
      "mlm": 1.5491628348827362,
      "distill": 1.0110661089420319,
      "teacher_acc": 0.7177830338478088
    },
    {
      "step": 9561,
      "lr": 1.8238220092457907e-05,
      "elapsed_s": 7103.389812469482,
      "loss": 2.964231550693512,
      "hidden": 0.18690602853894234,
      "layer": 0.09038912691175938,
      "mlm": 1.5221154391765594,
      "distill": 1.023109570145607,
      "teacher_acc": 0.7127853482961655
    },
    {
      "step": 9571,
      "lr": 1.818976429455227e-05,
      "elapsed_s": 7110.818918943405,
      "loss": 3.039660096168518,
      "hidden": 0.18311809748411179,
      "layer": 0.09001516364514828,
      "mlm": 1.5474238693714142,
      "distill": 1.0809924751520157,
      "teacher_acc": 0.7194809466600418
    },
    {
      "step": 9581,
      "lr": 1.8141336115938855e-05,
      "elapsed_s": 7118.218975543976,
      "loss": 2.8330073952674866,
      "hidden": 0.17944420129060745,
      "layer": 0.0872870646417141,
      "mlm": 1.4353662431240082,
      "distill": 0.9951091408729553,
      "teacher_acc": 0.7472753375768661
    },
    {
      "step": 9591,
      "lr": 1.8092935753020872e-05,
      "elapsed_s": 7125.602208852768,
      "loss": 2.857641875743866,
      "hidden": 0.18098565563559532,
      "layer": 0.08737301453948021,
      "mlm": 1.4529776573181152,
      "distill": 0.999006375670433,
      "teacher_acc": 0.736237108707428
    },
    {
      "step": 9601,
      "lr": 1.8044563402088684e-05,
      "elapsed_s": 7133.003538131714,
      "loss": 2.9869678020477295,
      "hidden": 0.18362656980752945,
      "layer": 0.08825022913515568,
      "mlm": 1.5269827842712402,
      "distill": 1.0486067533493042,
      "teacher_acc": 0.7159464806318283
    },
    {
      "step": 9611,
      "lr": 1.799621925931908e-05,
      "elapsed_s": 7140.358384132385,
      "loss": 2.891387164592743,
      "hidden": 0.180673535913229,
      "layer": 0.08755313791334629,
      "mlm": 1.4683393239974976,
      "distill": 1.017924278974533,
      "teacher_acc": 0.7327839434146881
    },
    {
      "step": 9621,
      "lr": 1.794790352077444e-05,
      "elapsed_s": 7147.730759143829,
      "loss": 2.8908419609069824,
      "hidden": 0.17931223288178444,
      "layer": 0.08794867619872093,
      "mlm": 1.491335928440094,
      "distill": 0.9969072341918945,
      "teacher_acc": 0.7340661883354187
    },
    {
      "step": 9631,
      "lr": 1.7899616382401936e-05,
      "elapsed_s": 7155.118606090546,
      "loss": 2.95628160238266,
      "hidden": 0.18000198900699615,
      "layer": 0.08795659244060516,
      "mlm": 1.5590976476669312,
      "distill": 0.9932016432285309,
      "teacher_acc": 0.70753613114357
    },
    {
      "step": 9641,
      "lr": 1.7851358040032774e-05,
      "elapsed_s": 7162.507756948471,
      "loss": 3.0132035613059998,
      "hidden": 0.18315986916422844,
      "layer": 0.08948343992233276,
      "mlm": 1.563630759716034,
      "distill": 1.0385113209486008,
      "teacher_acc": 0.7155729830265045
    },
    {
      "step": 9651,
      "lr": 1.7803128689381348e-05,
      "elapsed_s": 7169.892201662064,
      "loss": 2.875238835811615,
      "hidden": 0.18072212859988213,
      "layer": 0.08905250765383244,
      "mlm": 1.459745466709137,
      "distill": 1.0095228254795074,
      "teacher_acc": 0.730450227856636
    },
    {
      "step": 9661,
      "lr": 1.7754928526044508e-05,
      "elapsed_s": 7177.268668889999,
      "loss": 2.973810315132141,
      "hidden": 0.1847980059683323,
      "layer": 0.0908763688057661,
      "mlm": 1.5130847096443176,
      "distill": 1.045691356062889,
      "teacher_acc": 0.7287781536579132
    },
    {
      "step": 9671,
      "lr": 1.7706757745500704e-05,
      "elapsed_s": 7184.6558883190155,
      "loss": 2.923809051513672,
      "hidden": 0.17994924634695053,
      "layer": 0.0881443228572607,
      "mlm": 1.5359739661216736,
      "distill": 0.983864426612854,
      "teacher_acc": 0.7246852517127991
    },
    {
      "step": 9681,
      "lr": 1.7658616543109236e-05,
      "elapsed_s": 7192.055111885071,
      "loss": 3.047480881214142,
      "hidden": 0.1849786527454853,
      "layer": 0.09122404456138611,
      "mlm": 1.5645566284656525,
      "distill": 1.0673549324274063,
      "teacher_acc": 0.7143561840057373
    },
    {
      "step": 9691,
      "lr": 1.761050511410945e-05,
      "elapsed_s": 7199.431858301163,
      "loss": 2.89900541305542,
      "hidden": 0.1786932721734047,
      "layer": 0.08856531418859959,
      "mlm": 1.503677785396576,
      "distill": 0.9936584383249283,
      "teacher_acc": 0.7269929200410843
    },
    {
      "step": 9701,
      "lr": 1.756242365361993e-05,
      "elapsed_s": 7206.812950134277,
      "loss": 2.953303575515747,
      "hidden": 0.1790499947965145,
      "layer": 0.08835804834961891,
      "mlm": 1.5425972640514374,
      "distill": 1.0084273517131805,
      "teacher_acc": 0.7097474932670593
    },
    {
      "step": 9711,
      "lr": 1.7514372356637737e-05,
      "elapsed_s": 7214.2141535282135,
      "loss": 2.9644117951393127,
      "hidden": 0.18014828488230705,
      "layer": 0.08812434785068035,
      "mlm": 1.5744625627994537,
      "distill": 0.9855905473232269,
      "teacher_acc": 0.7067606449127197
    },
    {
      "step": 9721,
      "lr": 1.746635141803761e-05,
      "elapsed_s": 7221.654469490051,
      "loss": 2.835680067539215,
      "hidden": 0.18059510737657547,
      "layer": 0.08811885118484497,
      "mlm": 1.423556238412857,
      "distill": 1.0068742334842682,
      "teacher_acc": 0.7343530356884003
    },
    {
      "step": 9731,
      "lr": 1.741836103257113e-05,
      "elapsed_s": 7229.05019235611,
      "loss": 2.9229743480682373,
      "hidden": 0.17685290053486824,
      "layer": 0.08774774894118309,
      "mlm": 1.5588662922382355,
      "distill": 0.9665284156799316,
      "teacher_acc": 0.7013914734125137
    },
    {
      "step": 9741,
      "lr": 1.737040139486601e-05,
      "elapsed_s": 7236.461788415909,
      "loss": 2.9146392941474915,
      "hidden": 0.1778394728899002,
      "layer": 0.08685998804867268,
      "mlm": 1.544300228357315,
      "distill": 0.9712301343679428,
      "teacher_acc": 0.7115725874900818
    },
    {
      "step": 9751,
      "lr": 1.7322472699425234e-05,
      "elapsed_s": 7243.846519708633,
      "loss": 2.9185555577278137,
      "hidden": 0.17941498011350632,
      "layer": 0.08871552720665932,
      "mlm": 1.5069102048873901,
      "distill": 1.0084575861692429,
      "teacher_acc": 0.718881145119667
    },
    {
      "step": 9761,
      "lr": 1.7274575140626318e-05,
      "elapsed_s": 7251.268625736237,
      "loss": 2.7823609113693237,
      "hidden": 0.18060053139925003,
      "layer": 0.08777744695544243,
      "mlm": 1.3797779083251953,
      "distill": 0.9974932074546814,
      "teacher_acc": 0.7495899647474289
    },
    {
      "step": 9771,
      "lr": 1.7226708912720495e-05,
      "elapsed_s": 7258.690372467041,
      "loss": 2.975046753883362,
      "hidden": 0.17362254485487938,
      "layer": 0.08692162297666073,
      "mlm": 1.5620619058609009,
      "distill": 1.0222789943218231,
      "teacher_acc": 0.7179158478975296
    },
    {
      "step": 9781,
      "lr": 1.7178874209831926e-05,
      "elapsed_s": 7266.119971990585,
      "loss": 2.9299359917640686,
      "hidden": 0.18009553477168083,
      "layer": 0.08906017430126667,
      "mlm": 1.5294901132583618,
      "distill": 0.9957247525453568,
      "teacher_acc": 0.7166904807090759
    },
    {
      "step": 9791,
      "lr": 1.713107122595693e-05,
      "elapsed_s": 7273.535041809082,
      "loss": 2.927403748035431,
      "hidden": 0.1768750362098217,
      "layer": 0.08731957338750362,
      "mlm": 1.5450462102890015,
      "distill": 0.9849476665258408,
      "teacher_acc": 0.7107117027044296
    },
    {
      "step": 9801,
      "lr": 1.7083300154963193e-05,
      "elapsed_s": 7280.95568728447,
      "loss": 2.7772064805030823,
      "hidden": 0.17559348419308662,
      "layer": 0.08633293025195599,
      "mlm": 1.4432721138000488,
      "distill": 0.9395809322595596,
      "teacher_acc": 0.7306736409664154
    },
    {
      "step": 9811,
      "lr": 1.7035561190588952e-05,
      "elapsed_s": 7288.366719245911,
      "loss": 3.0731698274612427,
      "hidden": 0.1909678913652897,
      "layer": 0.09065776504576206,
      "mlm": 1.5138513147830963,
      "distill": 1.1320538520812988,
      "teacher_acc": 0.7327067255973816
    },
    {
      "step": 9821,
      "lr": 1.6987854526442264e-05,
      "elapsed_s": 7295.805382728577,
      "loss": 2.967125654220581,
      "hidden": 0.1809689849615097,
      "layer": 0.08931695856153965,
      "mlm": 1.5038708448410034,
      "distill": 1.0566583424806595,
      "teacher_acc": 0.734115868806839
    },
    {
      "step": 9831,
      "lr": 1.6940180356000168e-05,
      "elapsed_s": 7303.246222734451,
      "loss": 2.9639387130737305,
      "hidden": 0.18302374705672264,
      "layer": 0.0889599658548832,
      "mlm": 1.4987303912639618,
      "distill": 1.0546807646751404,
      "teacher_acc": 0.7253968268632889
    },
    {
      "step": 9841,
      "lr": 1.6892538872607937e-05,
      "elapsed_s": 7310.652861595154,
      "loss": 2.844963312149048,
      "hidden": 0.17552710697054863,
      "layer": 0.08675637282431126,
      "mlm": 1.4993855357170105,
      "distill": 0.9511454403400421,
      "teacher_acc": 0.71675243973732
    },
    {
      "step": 9851,
      "lr": 1.6844930269478274e-05,
      "elapsed_s": 7318.030217170715,
      "loss": 2.8490097522735596,
      "hidden": 0.17603646218776703,
      "layer": 0.08703974261879921,
      "mlm": 1.504111260175705,
      "distill": 0.9493057131767273,
      "teacher_acc": 0.7179317623376846
    },
    {
      "step": 9861,
      "lr": 1.679735473969053e-05,
      "elapsed_s": 7325.405548334122,
      "loss": 2.9066742062568665,
      "hidden": 0.18075070902705193,
      "layer": 0.08813525177538395,
      "mlm": 1.5162778794765472,
      "distill": 0.9848272651433945,
      "teacher_acc": 0.7127037644386292
    },
    {
      "step": 9871,
      "lr": 1.6749812476189935e-05,
      "elapsed_s": 7332.798872947693,
      "loss": 3.1854604482650757,
      "hidden": 0.18603482469916344,
      "layer": 0.09088591858744621,
      "mlm": 1.558554083108902,
      "distill": 1.2093938887119293,
      "teacher_acc": 0.7316144406795502
    },
    {
      "step": 9881,
      "lr": 1.6702303671786797e-05,
      "elapsed_s": 7340.163810968399,
      "loss": 2.8082371950149536,
      "hidden": 0.18210791796445847,
      "layer": 0.08892356231808662,
      "mlm": 1.3525468707084656,
      "distill": 1.0470127761363983,
      "teacher_acc": 0.7591568678617477
    },
    {
      "step": 9891,
      "lr": 1.665482851915573e-05,
      "elapsed_s": 7347.554973125458,
      "loss": 2.887851655483246,
      "hidden": 0.1780196614563465,
      "layer": 0.08653615042567253,
      "mlm": 1.493287056684494,
      "distill": 0.9952571094036102,
      "teacher_acc": 0.7202744483947754
    },
    {
      "step": 9901,
      "lr": 1.6607387210834887e-05,
      "elapsed_s": 7354.904918670654,
      "loss": 3.0758196115493774,
      "hidden": 0.1849890537559986,
      "layer": 0.08875576965510845,
      "mlm": 1.647627055644989,
      "distill": 1.0138365626335144,
      "teacher_acc": 0.6910272836685181
    },
    {
      "step": 9911,
      "lr": 1.6559979939225128e-05,
      "elapsed_s": 7362.275346040726,
      "loss": 2.8595410585403442,
      "hidden": 0.17881251499056816,
      "layer": 0.08778204023838043,
      "mlm": 1.4244898855686188,
      "distill": 1.033535212278366,
      "teacher_acc": 0.7343051582574844
    },
    {
      "step": 9921,
      "lr": 1.6512606896589323e-05,
      "elapsed_s": 7369.682178497314,
      "loss": 2.8354130387306213,
      "hidden": 0.17696453258395195,
      "layer": 0.08690304309129715,
      "mlm": 1.5012941658496857,
      "distill": 0.9367383569478989,
      "teacher_acc": 0.7208409160375595
    },
    {
      "step": 9931,
      "lr": 1.646526827505148e-05,
      "elapsed_s": 7377.0850830078125,
      "loss": 2.8742676377296448,
      "hidden": 0.17765381932258606,
      "layer": 0.08699697442352772,
      "mlm": 1.4730963706970215,
      "distill": 1.002365157008171,
      "teacher_acc": 0.7427856922149658
    },
    {
      "step": 9941,
      "lr": 1.6417964266596043e-05,
      "elapsed_s": 7384.455932855606,
      "loss": 2.9628148674964905,
      "hidden": 0.18117588013410568,
      "layer": 0.08969128876924515,
      "mlm": 1.5167653858661652,
      "distill": 1.0388521254062653,
      "teacher_acc": 0.7283293008804321
    },
    {
      "step": 9951,
      "lr": 1.6370695063067053e-05,
      "elapsed_s": 7391.832479476929,
      "loss": 2.851147711277008,
      "hidden": 0.17699117958545685,
      "layer": 0.08680770173668861,
      "mlm": 1.4818080067634583,
      "distill": 0.9719534516334534,
      "teacher_acc": 0.7335547059774399
    },
    {
      "step": 9961,
      "lr": 1.6323460856167426e-05,
      "elapsed_s": 7399.206396579742,
      "loss": 3.0169769525527954,
      "hidden": 0.18121608719229698,
      "layer": 0.08835496753454208,
      "mlm": 1.5947742760181427,
      "distill": 1.0155929774045944,
      "teacher_acc": 0.7150907069444656
    },
    {
      "step": 9971,
      "lr": 1.6276261837458123e-05,
      "elapsed_s": 7406.59486746788,
      "loss": 2.8825445771217346,
      "hidden": 0.1777927614748478,
      "layer": 0.0882922112941742,
      "mlm": 1.4628415405750275,
      "distill": 1.0199714004993439,
      "teacher_acc": 0.7431914657354355
    },
    {
      "step": 9981,
      "lr": 1.6229098198357408e-05,
      "elapsed_s": 7413.9206635952,
      "loss": 2.938017189502716,
      "hidden": 0.1840236410498619,
      "layer": 0.0877886414527893,
      "mlm": 1.53719824552536,
      "distill": 0.9888773113489151,
      "teacher_acc": 0.7157649099826813
    },
    {
      "step": 9991,
      "lr": 1.6181970130140066e-05,
      "elapsed_s": 7421.304645776749,
      "loss": 2.80196076631546,
      "hidden": 0.1804322525858879,
      "layer": 0.08815579116344452,
      "mlm": 1.4045225381851196,
      "distill": 0.9924958199262619,
      "teacher_acc": 0.7442905008792877
    },
    {
      "step": 10001,
      "lr": 1.613487782393661e-05,
      "elapsed_s": 7430.69629073143,
      "loss": 3.003439486026764,
      "hidden": 0.17982863262295723,
      "layer": 0.08787352591753006,
      "mlm": 1.6038479506969452,
      "distill": 0.9959975630044937,
      "teacher_acc": 0.7019294500350952
    },
    {
      "step": 10011,
      "lr": 1.608782147073253e-05,
      "elapsed_s": 7438.061214208603,
      "loss": 3.0181021690368652,
      "hidden": 0.1852869726717472,
      "layer": 0.08996334671974182,
      "mlm": 1.5328517258167267,
      "distill": 1.0696948766708374,
      "teacher_acc": 0.7213388383388519
    },
    {
      "step": 10021,
      "lr": 1.6040801261367493e-05,
      "elapsed_s": 7445.437765598297,
      "loss": 2.9223610162734985,
      "hidden": 0.1778326965868473,
      "layer": 0.08830015361309052,
      "mlm": 1.5157210528850555,
      "distill": 1.006824478507042,
      "teacher_acc": 0.7290975898504257
    },
    {
      "step": 10031,
      "lr": 1.5993817386534597e-05,
      "elapsed_s": 7452.824325323105,
      "loss": 2.8421236276626587,
      "hidden": 0.17729509249329567,
      "layer": 0.08825144357979298,
      "mlm": 1.460149198770523,
      "distill": 0.9832584708929062,
      "teacher_acc": 0.7264283001422882
    },
    {
      "step": 10041,
      "lr": 1.5946870036779586e-05,
      "elapsed_s": 7460.190210819244,
      "loss": 2.9949209690093994,
      "hidden": 0.18199637159705162,
      "layer": 0.08990189991891384,
      "mlm": 1.5370220839977264,
      "distill": 1.0489552170038223,
      "teacher_acc": 0.7236683368682861
    },
    {
      "step": 10051,
      "lr": 1.5899959402500046e-05,
      "elapsed_s": 7467.531365156174,
      "loss": 2.805436432361603,
      "hidden": 0.18041185662150383,
      "layer": 0.08785103633999825,
      "mlm": 1.4099665582180023,
      "distill": 0.9907206147909164,
      "teacher_acc": 0.7366474866867065
    },
    {
      "step": 10061,
      "lr": 1.5853085673944697e-05,
      "elapsed_s": 7474.873639345169,
      "loss": 2.9663367867469788,
      "hidden": 0.17970432713627815,
      "layer": 0.08887140452861786,
      "mlm": 1.5310087203979492,
      "distill": 1.0314836949110031,
      "teacher_acc": 0.7196492105722427
    },
    {
      "step": 10071,
      "lr": 1.5806249041212563e-05,
      "elapsed_s": 7482.232886552811,
      "loss": 2.9648497700691223,
      "hidden": 0.17497874423861504,
      "layer": 0.08719329908490181,
      "mlm": 1.5084562003612518,
      "distill": 1.0628393441438675,
      "teacher_acc": 0.7249744683504105
    },
    {
      "step": 10081,
      "lr": 1.5759449694252226e-05,
      "elapsed_s": 7489.616062641144,
      "loss": 2.838091492652893,
      "hidden": 0.17410090565681458,
      "layer": 0.08699353970587254,
      "mlm": 1.5020561516284943,
      "distill": 0.9443367123603821,
      "teacher_acc": 0.7131647020578384
    },
    {
      "step": 10091,
      "lr": 1.571268782286107e-05,
      "elapsed_s": 7497.00991153717,
      "loss": 2.9380322694778442,
      "hidden": 0.18527715653181076,
      "layer": 0.08950702846050262,
      "mlm": 1.449166625738144,
      "distill": 1.0735578536987305,
      "teacher_acc": 0.7430115938186646
    },
    {
      "step": 10101,
      "lr": 1.5665963616684476e-05,
      "elapsed_s": 7504.36677980423,
      "loss": 2.930846869945526,
      "hidden": 0.1796138398349285,
      "layer": 0.08817419223487377,
      "mlm": 1.5123560428619385,
      "distill": 1.0151760578155518,
      "teacher_acc": 0.7207438051700592
    },
    {
      "step": 10111,
      "lr": 1.5619277265215077e-05,
      "elapsed_s": 7511.744881391525,
      "loss": 2.921996772289276,
      "hidden": 0.1769735887646675,
      "layer": 0.08746769651770592,
      "mlm": 1.5687729120254517,
      "distill": 0.9555428922176361,
      "teacher_acc": 0.7002024799585342
    },
    {
      "step": 10121,
      "lr": 1.557262895779199e-05,
      "elapsed_s": 7519.128073215485,
      "loss": 2.852729916572571,
      "hidden": 0.17312494292855263,
      "layer": 0.08698642253875732,
      "mlm": 1.5092523694038391,
      "distill": 0.953734427690506,
      "teacher_acc": 0.7120293378829956
    },
    {
      "step": 10131,
      "lr": 1.5526018883600037e-05,
      "elapsed_s": 7526.520139694214,
      "loss": 2.827915370464325,
      "hidden": 0.1762291006743908,
      "layer": 0.08715229481458664,
      "mlm": 1.4780460894107819,
      "distill": 0.9538348913192749,
      "teacher_acc": 0.726918026804924
    },
    {
      "step": 10141,
      "lr": 1.5479447231668978e-05,
      "elapsed_s": 7533.927247285843,
      "loss": 2.983986258506775,
      "hidden": 0.18912728503346443,
      "layer": 0.09127629175782204,
      "mlm": 1.466661125421524,
      "distill": 1.0934324264526367,
      "teacher_acc": 0.74553382396698
    },
    {
      "step": 10151,
      "lr": 1.5432914190872757e-05,
      "elapsed_s": 7541.360129594803,
      "loss": 2.7671077251434326,
      "hidden": 0.1783139854669571,
      "layer": 0.0875600166618824,
      "mlm": 1.4014981091022491,
      "distill": 0.965201660990715,
      "teacher_acc": 0.7436766773462296
    },
    {
      "step": 10161,
      "lr": 1.5386419949928732e-05,
      "elapsed_s": 7548.769009590149,
      "loss": 2.8664923906326294,
      "hidden": 0.17761590331792831,
      "layer": 0.08717396482825279,
      "mlm": 1.4904872477054596,
      "distill": 0.9771863222122192,
      "teacher_acc": 0.7179876863956451
    },
    {
      "step": 10171,
      "lr": 1.5339964697396897e-05,
      "elapsed_s": 7556.171970129013,
      "loss": 2.9887197017669678,
      "hidden": 0.18235337734222412,
      "layer": 0.08834496513009071,
      "mlm": 1.553718626499176,
      "distill": 1.0261218696832657,
      "teacher_acc": 0.7176017761230469
    },
    {
      "step": 10181,
      "lr": 1.5293548621679127e-05,
      "elapsed_s": 7563.583142518997,
      "loss": 2.894451141357422,
      "hidden": 0.17966174706816673,
      "layer": 0.08759517222642899,
      "mlm": 1.5086528062820435,
      "distill": 0.9826772511005402,
      "teacher_acc": 0.7285481691360474
    },
    {
      "step": 10191,
      "lr": 1.5247171911018427e-05,
      "elapsed_s": 7571.021494150162,
      "loss": 2.9824821949005127,
      "hidden": 0.1800749935209751,
      "layer": 0.08921219408512115,
      "mlm": 1.5568162202835083,
      "distill": 1.020909920334816,
      "teacher_acc": 0.7141296565532684
    },
    {
      "step": 10201,
      "lr": 1.5200834753498128e-05,
      "elapsed_s": 7578.43163728714,
      "loss": 2.9277824759483337,
      "hidden": 0.1826518028974533,
      "layer": 0.08851532824337482,
      "mlm": 1.5152919590473175,
      "distill": 1.0029292851686478,
      "teacher_acc": 0.7173645943403244
    },
    {
      "step": 10211,
      "lr": 1.5154537337041174e-05,
      "elapsed_s": 7585.876492738724,
      "loss": 2.8564310669898987,
      "hidden": 0.18232426792383194,
      "layer": 0.0884082205593586,
      "mlm": 1.4360493123531342,
      "distill": 1.0115291327238083,
      "teacher_acc": 0.7387486547231674
    },
    {
      "step": 10221,
      "lr": 1.5108279849409335e-05,
      "elapsed_s": 7593.2914979457855,
      "loss": 2.906259834766388,
      "hidden": 0.18123461678624153,
      "layer": 0.08788170851767063,
      "mlm": 1.5110745131969452,
      "distill": 0.9887753427028656,
      "teacher_acc": 0.7239549160003662
    },
    {
      "step": 10231,
      "lr": 1.5062062478202436e-05,
      "elapsed_s": 7600.722090005875,
      "loss": 2.947192668914795,
      "hidden": 0.17599565163254738,
      "layer": 0.08744473196566105,
      "mlm": 1.557024359703064,
      "distill": 0.9944546073675156,
      "teacher_acc": 0.70560422539711
    },
    {
      "step": 10241,
      "lr": 1.5015885410857616e-05,
      "elapsed_s": 7608.113600492477,
      "loss": 2.8559728860855103,
      "hidden": 0.18119856715202332,
      "layer": 0.08960779383778572,
      "mlm": 1.417513757944107,
      "distill": 1.0312581360340118,
      "teacher_acc": 0.7418114244937897
    },
    {
      "step": 10251,
      "lr": 1.4969748834648556e-05,
      "elapsed_s": 7615.547703504562,
      "loss": 2.7950891256332397,
      "hidden": 0.17494433000683784,
      "layer": 0.08769348077476025,
      "mlm": 1.396903246641159,
      "distill": 1.004450410604477,
      "teacher_acc": 0.7413361072540283
    },
    {
      "step": 10261,
      "lr": 1.4923652936684723e-05,
      "elapsed_s": 7622.975331783295,
      "loss": 2.853096663951874,
      "hidden": 0.17926686629652977,
      "layer": 0.08738799579441547,
      "mlm": 1.4810059666633606,
      "distill": 0.9698629826307297,
      "teacher_acc": 0.7235698997974396
    },
    {
      "step": 10271,
      "lr": 1.4877597903910617e-05,
      "elapsed_s": 7630.39523267746,
      "loss": 2.9036999940872192,
      "hidden": 0.17764588445425034,
      "layer": 0.08864590153098106,
      "mlm": 1.4913431704044342,
      "distill": 1.012742057442665,
      "teacher_acc": 0.7309898138046265
    },
    {
      "step": 10281,
      "lr": 1.4831583923104999e-05,
      "elapsed_s": 7637.821432590485,
      "loss": 2.9167041182518005,
      "hidden": 0.18047721683979034,
      "layer": 0.08668654225766659,
      "mlm": 1.5306980907917023,
      "distill": 0.9817083179950714,
      "teacher_acc": 0.7128436267375946
    },
    {
      "step": 10291,
      "lr": 1.4785611180880143e-05,
      "elapsed_s": 7645.225907325745,
      "loss": 3.012228846549988,
      "hidden": 0.18359127268195152,
      "layer": 0.08894979394972324,
      "mlm": 1.533097267150879,
      "distill": 1.067474126815796,
      "teacher_acc": 0.7218306511640549
    },
    {
      "step": 10301,
      "lr": 1.4739679863681086e-05,
      "elapsed_s": 7652.653679847717,
      "loss": 2.917469322681427,
      "hidden": 0.17814286053180695,
      "layer": 0.08777454309165478,
      "mlm": 1.504893273115158,
      "distill": 1.0124030709266663,
      "teacher_acc": 0.7199055850505829
    },
    {
      "step": 10311,
      "lr": 1.4693790157784848e-05,
      "elapsed_s": 7660.067158937454,
      "loss": 2.8820289373397827,
      "hidden": 0.1813196875154972,
      "layer": 0.08812155947089195,
      "mlm": 1.4435787498950958,
      "distill": 1.031750038266182,
      "teacher_acc": 0.7325315773487091
    },
    {
      "step": 10321,
      "lr": 1.4647942249299707e-05,
      "elapsed_s": 7667.485271930695,
      "loss": 2.9324496388435364,
      "hidden": 0.17789708077907562,
      "layer": 0.0877643059939146,
      "mlm": 1.4990541338920593,
      "distill": 1.0337191224098206,
      "teacher_acc": 0.7291924357414246
    },
    {
      "step": 10331,
      "lr": 1.4602136324164417e-05,
      "elapsed_s": 7674.917230606079,
      "loss": 2.8862799406051636,
      "hidden": 0.17475510761141777,
      "layer": 0.08737952448427677,
      "mlm": 1.5169754028320312,
      "distill": 0.9761045724153519,
      "teacher_acc": 0.7264734357595444
    },
    {
      "step": 10341,
      "lr": 1.4556372568147483e-05,
      "elapsed_s": 7682.341321468353,
      "loss": 3.048863112926483,
      "hidden": 0.1806645803153515,
      "layer": 0.08840730786323547,
      "mlm": 1.5391749441623688,
      "distill": 1.1041553914546967,
      "teacher_acc": 0.7199923396110535
    },
    {
      "step": 10351,
      "lr": 1.4510651166846367e-05,
      "elapsed_s": 7689.776777982712,
      "loss": 2.903261661529541,
      "hidden": 0.18201984837651253,
      "layer": 0.08788246847689152,
      "mlm": 1.5032626688480377,
      "distill": 0.9920180886983871,
      "teacher_acc": 0.7213344126939774
    },
    {
      "step": 10361,
      "lr": 1.4464972305686777e-05,
      "elapsed_s": 7697.223017215729,
      "loss": 2.8741148710250854,
      "hidden": 0.18011771887540817,
      "layer": 0.08773745037615299,
      "mlm": 1.4926083981990814,
      "distill": 0.9774023294448853,
      "teacher_acc": 0.7121809720993042
    },
    {
      "step": 10371,
      "lr": 1.4419336169921893e-05,
      "elapsed_s": 7704.64383482933,
      "loss": 2.858348250389099,
      "hidden": 0.17916475608944893,
      "layer": 0.08749309927225113,
      "mlm": 1.4723995625972748,
      "distill": 0.9838726371526718,
      "teacher_acc": 0.7229121923446655
    },
    {
      "step": 10381,
      "lr": 1.4373742944631622e-05,
      "elapsed_s": 7712.074449300766,
      "loss": 2.917595684528351,
      "hidden": 0.18145322427153587,
      "layer": 0.08780582807958126,
      "mlm": 1.5095360279083252,
      "distill": 1.0012503117322922,
      "teacher_acc": 0.7191878110170364
    },
    {
      "step": 10391,
      "lr": 1.4328192814721846e-05,
      "elapsed_s": 7719.528669595718,
      "loss": 2.80202579498291,
      "hidden": 0.17872299253940582,
      "layer": 0.08836937882006168,
      "mlm": 1.4123475551605225,
      "distill": 0.9880475699901581,
      "teacher_acc": 0.742987871170044
    },
    {
      "step": 10401,
      "lr": 1.4282685964923642e-05,
      "elapsed_s": 7726.955592632294,
      "loss": 2.9938812255859375,
      "hidden": 0.1879442222416401,
      "layer": 0.09049897082149982,
      "mlm": 1.4676394164562225,
      "distill": 1.1051038056612015,
      "teacher_acc": 0.7386718243360519
    },
    {
      "step": 10411,
      "lr": 1.4237222579792618e-05,
      "elapsed_s": 7734.3602204322815,
      "loss": 2.807776629924774,
      "hidden": 0.17281070351600647,
      "layer": 0.08571877330541611,
      "mlm": 1.492429256439209,
      "distill": 0.9268665760755539,
      "teacher_acc": 0.7185997068881989
    },
    {
      "step": 10421,
      "lr": 1.4191802843708062e-05,
      "elapsed_s": 7741.776548862457,
      "loss": 2.9173110127449036,
      "hidden": 0.18102671578526497,
      "layer": 0.08790738321840763,
      "mlm": 1.528998464345932,
      "distill": 0.9823053926229477,
      "teacher_acc": 0.7239071279764175
    },
    {
      "step": 10431,
      "lr": 1.4146426940872277e-05,
      "elapsed_s": 7749.188870191574,
      "loss": 2.8313148617744446,
      "hidden": 0.18319915235042572,
      "layer": 0.08748494647443295,
      "mlm": 1.4635584354400635,
      "distill": 0.9576156586408615,
      "teacher_acc": 0.7208824902772903
    },
    {
      "step": 10441,
      "lr": 1.4101095055309746e-05,
      "elapsed_s": 7756.6106152534485,
      "loss": 2.92011559009552,
      "hidden": 0.1835278421640396,
      "layer": 0.08800114318728447,
      "mlm": 1.5004338026046753,
      "distill": 1.0086255371570587,
      "teacher_acc": 0.7249470055103302
    },
    {
      "step": 10451,
      "lr": 1.4055807370866485e-05,
      "elapsed_s": 7764.054565906525,
      "loss": 2.8488921523094177,
      "hidden": 0.1790946125984192,
      "layer": 0.08824432268738747,
      "mlm": 1.4330228865146637,
      "distill": 1.013557881116867,
      "teacher_acc": 0.7401723563671112
    },
    {
      "step": 10461,
      "lr": 1.4010564071209237e-05,
      "elapsed_s": 7771.451988220215,
      "loss": 2.8307689428329468,
      "hidden": 0.1771230474114418,
      "layer": 0.08662055991590023,
      "mlm": 1.4766446352005005,
      "distill": 0.9565679430961609,
      "teacher_acc": 0.7279039025306702
    },
    {
      "step": 10471,
      "lr": 1.3965365339824732e-05,
      "elapsed_s": 7778.867230892181,
      "loss": 2.9182360768318176,
      "hidden": 0.1831325888633728,
      "layer": 0.08828550390899181,
      "mlm": 1.4922117590904236,
      "distill": 1.0156163573265076,
      "teacher_acc": 0.7282605767250061
    },
    {
      "step": 10481,
      "lr": 1.392021136001897e-05,
      "elapsed_s": 7786.289425611496,
      "loss": 2.8439837098121643,
      "hidden": 0.17795449122786522,
      "layer": 0.08859201148152351,
      "mlm": 1.4827484786510468,
      "distill": 0.9610302448272705,
      "teacher_acc": 0.717214897274971
    },
    {
      "step": 10491,
      "lr": 1.3875102314916422e-05,
      "elapsed_s": 7793.672843456268,
      "loss": 2.9404280185699463,
      "hidden": 0.1754220873117447,
      "layer": 0.08772540651261806,
      "mlm": 1.539043664932251,
      "distill": 1.0066774785518646,
      "teacher_acc": 0.7314690053462982
    },
    {
      "step": 10501,
      "lr": 1.3830038387459354e-05,
      "elapsed_s": 7801.119505882263,
      "loss": 2.8838931918144226,
      "hidden": 0.17829493805766106,
      "layer": 0.08872641064226627,
      "mlm": 1.480123370885849,
      "distill": 1.0028167814016342,
      "teacher_acc": 0.7373668849468231
    },
    {
      "step": 10511,
      "lr": 1.3785019760407053e-05,
      "elapsed_s": 7808.514775514603,
      "loss": 2.714491546154022,
      "hidden": 0.174835667014122,
      "layer": 0.08605832792818546,
      "mlm": 1.3704897165298462,
      "distill": 0.9513013362884521,
      "teacher_acc": 0.7451911568641663
    },
    {
      "step": 10521,
      "lr": 1.3740046616335079e-05,
      "elapsed_s": 7815.936735391617,
      "loss": 2.73585307598114,
      "hidden": 0.17594927921891212,
      "layer": 0.08653202652931213,
      "mlm": 1.3991585075855255,
      "distill": 0.9415299445390701,
      "teacher_acc": 0.7395884245634079
    },
    {
      "step": 10531,
      "lr": 1.3695119137634544e-05,
      "elapsed_s": 7823.345595598221,
      "loss": 2.9362568259239197,
      "hidden": 0.18131597340106964,
      "layer": 0.0884503498673439,
      "mlm": 1.5110560953617096,
      "distill": 1.0183437019586563,
      "teacher_acc": 0.7218842655420303
    },
    {
      "step": 10541,
      "lr": 1.3650237506511331e-05,
      "elapsed_s": 7830.731982707977,
      "loss": 2.8972848057746887,
      "hidden": 0.1792755089700222,
      "layer": 0.08849368244409561,
      "mlm": 1.4782765209674835,
      "distill": 1.0162105113267899,
      "teacher_acc": 0.7314103543758392
    },
    {
      "step": 10551,
      "lr": 1.3605401904985427e-05,
      "elapsed_s": 7838.131356954575,
      "loss": 3.0389745235443115,
      "hidden": 0.18006238713860512,
      "layer": 0.08841749839484692,
      "mlm": 1.5982315242290497,
      "distill": 1.0364094972610474,
      "teacher_acc": 0.7085699439048767
    },
    {
      "step": 10561,
      "lr": 1.356061251489012e-05,
      "elapsed_s": 7845.543134689331,
      "loss": 2.9946030378341675,
      "hidden": 0.18319226801395416,
      "layer": 0.09013243392109871,
      "mlm": 1.528556913137436,
      "distill": 1.0545953661203384,
      "teacher_acc": 0.7218819558620453
    },
    {
      "step": 10571,
      "lr": 1.3515869517871288e-05,
      "elapsed_s": 7852.972601175308,
      "loss": 2.9079365730285645,
      "hidden": 0.17865262180566788,
      "layer": 0.0872088111937046,
      "mlm": 1.462127685546875,
      "distill": 1.0448992252349854,
      "teacher_acc": 0.7356737852096558
    },
    {
      "step": 10581,
      "lr": 1.3471173095386669e-05,
      "elapsed_s": 7860.408521652222,
      "loss": 2.9706795811653137,
      "hidden": 0.18146351352334023,
      "layer": 0.08906749822199345,
      "mlm": 1.5658102929592133,
      "distill": 0.997408539056778,
      "teacher_acc": 0.718964233994484
    },
    {
      "step": 10591,
      "lr": 1.342652342870509e-05,
      "elapsed_s": 7867.839487552643,
      "loss": 2.820731282234192,
      "hidden": 0.17802704498171806,
      "layer": 0.08731149137020111,
      "mlm": 1.4431924521923065,
      "distill": 0.9778289347887039,
      "teacher_acc": 0.7360071837902069
    },
    {
      "step": 10601,
      "lr": 1.3381920698905787e-05,
      "elapsed_s": 7875.25711607933,
      "loss": 2.8756213188171387,
      "hidden": 0.1747138872742653,
      "layer": 0.08734213374555111,
      "mlm": 1.5557876825332642,
      "distill": 0.9267347753047943,
      "teacher_acc": 0.7119696736335754
    },
    {
      "step": 10611,
      "lr": 1.3337365086877623e-05,
      "elapsed_s": 7882.685705423355,
      "loss": 3.003625810146332,
      "hidden": 0.1858767867088318,
      "layer": 0.08926649950444698,
      "mlm": 1.5266430079936981,
      "distill": 1.0605959594249725,
      "teacher_acc": 0.7280881404876709
    },
    {
      "step": 10621,
      "lr": 1.3292856773318384e-05,
      "elapsed_s": 7890.094223737717,
      "loss": 2.8903027772903442,
      "hidden": 0.18244284763932228,
      "layer": 0.08799527771770954,
      "mlm": 1.504819631576538,
      "distill": 0.9765997678041458,
      "teacher_acc": 0.7186618745326996
    },
    {
      "step": 10631,
      "lr": 1.3248395938734032e-05,
      "elapsed_s": 7897.527639865875,
      "loss": 3.049826979637146,
      "hidden": 0.18883823975920677,
      "layer": 0.0903621856123209,
      "mlm": 1.5401052832603455,
      "distill": 1.086864173412323,
      "teacher_acc": 0.7175163477659225
    },
    {
      "step": 10641,
      "lr": 1.320398276343795e-05,
      "elapsed_s": 7904.978014707565,
      "loss": 2.7387565970420837,
      "hidden": 0.17387425154447556,
      "layer": 0.08550853095948696,
      "mlm": 1.3985196948051453,
      "distill": 0.9497341066598892,
      "teacher_acc": 0.7384101450443268
    },
    {
      "step": 10651,
      "lr": 1.315961742755027e-05,
      "elapsed_s": 7912.4241461753845,
      "loss": 2.7505109906196594,
      "hidden": 0.1737189069390297,
      "layer": 0.0857369601726532,
      "mlm": 1.4088835716247559,
      "distill": 0.9513211250305176,
      "teacher_acc": 0.7335278689861298
    },
    {
      "step": 10661,
      "lr": 1.3115300110997098e-05,
      "elapsed_s": 7919.868975877762,
      "loss": 2.897267520427704,
      "hidden": 0.17698285728693008,
      "layer": 0.08742825873196125,
      "mlm": 1.4780522882938385,
      "distill": 1.0215354710817337,
      "teacher_acc": 0.7327415645122528
    },
    {
      "step": 10671,
      "lr": 1.3071030993509788e-05,
      "elapsed_s": 7927.312104225159,
      "loss": 2.87199330329895,
      "hidden": 0.17923161014914513,
      "layer": 0.08656316436827183,
      "mlm": 1.489340603351593,
      "distill": 0.9809079021215439,
      "teacher_acc": 0.7273224741220474
    },
    {
      "step": 10681,
      "lr": 1.302681025462424e-05,
      "elapsed_s": 7934.736119031906,
      "loss": 2.896890938282013,
      "hidden": 0.17528121918439865,
      "layer": 0.08683363534510136,
      "mlm": 1.481823354959488,
      "distill": 1.0210883617401123,
      "teacher_acc": 0.7294953763484955
    },
    {
      "step": 10691,
      "lr": 1.2982638073680112e-05,
      "elapsed_s": 7942.174689292908,
      "loss": 2.759673297405243,
      "hidden": 0.17198539525270462,
      "layer": 0.08498093672096729,
      "mlm": 1.4453786313533783,
      "distill": 0.9278333932161331,
      "teacher_acc": 0.7291694581508636
    },
    {
      "step": 10701,
      "lr": 1.293851462982017e-05,
      "elapsed_s": 7949.615513324738,
      "loss": 2.9145883321762085,
      "hidden": 0.18482211977243423,
      "layer": 0.09008979983627796,
      "mlm": 1.4679408073425293,
      "distill": 1.0319584012031555,
      "teacher_acc": 0.736321285367012
    },
    {
      "step": 10711,
      "lr": 1.289444010198951e-05,
      "elapsed_s": 7957.037538290024,
      "loss": 3.049929201602936,
      "hidden": 0.18432985991239548,
      "layer": 0.08944540098309517,
      "mlm": 1.5419031381607056,
      "distill": 1.0946436524391174,
      "teacher_acc": 0.7228672504425049
    },
    {
      "step": 10721,
      "lr": 1.285041466893485e-05,
      "elapsed_s": 7964.456969976425,
      "loss": 2.964224874973297,
      "hidden": 0.17951473966240883,
      "layer": 0.0875111073255539,
      "mlm": 1.5440576076507568,
      "distill": 1.0173822790384293,
      "teacher_acc": 0.7159728109836578
    },
    {
      "step": 10731,
      "lr": 1.2806438509203803e-05,
      "elapsed_s": 7971.910104751587,
      "loss": 3.0077428817749023,
      "hidden": 0.186091348528862,
      "layer": 0.08951134793460369,
      "mlm": 1.5429365932941437,
      "distill": 1.0478679239749908,
      "teacher_acc": 0.7214069217443466
    },
    {
      "step": 10741,
      "lr": 1.2762511801144133e-05,
      "elapsed_s": 7979.328640222549,
      "loss": 2.91107714176178,
      "hidden": 0.1789752058684826,
      "layer": 0.08725208602845669,
      "mlm": 1.5028254389762878,
      "distill": 1.0066752284765244,
      "teacher_acc": 0.7244965136051178
    },
    {
      "step": 10751,
      "lr": 1.2718634722903073e-05,
      "elapsed_s": 7986.758846282959,
      "loss": 2.849497675895691,
      "hidden": 0.17816199362277985,
      "layer": 0.08739409036934376,
      "mlm": 1.4631358981132507,
      "distill": 0.9863407611846924,
      "teacher_acc": 0.7267728447914124
    },
    {
      "step": 10761,
      "lr": 1.2674807452426565e-05,
      "elapsed_s": 7994.169144392014,
      "loss": 2.9450109004974365,
      "hidden": 0.17648660764098167,
      "layer": 0.08721574582159519,
      "mlm": 1.5794736742973328,
      "distill": 0.9689561575651169,
      "teacher_acc": 0.7112008631229401
    },
    {
      "step": 10771,
      "lr": 1.2631030167458568e-05,
      "elapsed_s": 8001.56565785408,
      "loss": 2.9443331956863403,
      "hidden": 0.17649906873703003,
      "layer": 0.08623652346432209,
      "mlm": 1.5921371579170227,
      "distill": 0.9560795873403549,
      "teacher_acc": 0.7092689871788025
    },
    {
      "step": 10781,
      "lr": 1.2587303045540314e-05,
      "elapsed_s": 8009.01592540741,
      "loss": 2.8592692017555237,
      "hidden": 0.17810314521193504,
      "layer": 0.08709154278039932,
      "mlm": 1.4945299625396729,
      "distill": 0.9649871289730072,
      "teacher_acc": 0.7186283767223358
    },
    {
      "step": 10791,
      "lr": 1.2543626264009577e-05,
      "elapsed_s": 8016.4046857357025,
      "loss": 3.045073986053467,
      "hidden": 0.18405351415276527,
      "layer": 0.08889592066407204,
      "mlm": 1.5948028862476349,
      "distill": 1.037716120481491,
      "teacher_acc": 0.7050633430480957
    },
    {
      "step": 10801,
      "lr": 1.2500000000000006e-05,
      "elapsed_s": 8023.803291082382,
      "loss": 2.8610616326332092,
      "hidden": 0.17923183739185333,
      "layer": 0.08687343075871468,
      "mlm": 1.4446918070316315,
      "distill": 1.0144694596529007,
      "teacher_acc": 0.7359804809093475
    },
    {
      "step": 10811,
      "lr": 1.245642443044035e-05,
      "elapsed_s": 8031.231106758118,
      "loss": 2.919489026069641,
      "hidden": 0.1818200834095478,
      "layer": 0.08892287313938141,
      "mlm": 1.4692298769950867,
      "distill": 1.0421575605869293,
      "teacher_acc": 0.7311240881681442
    },
    {
      "step": 10821,
      "lr": 1.2412899732053773e-05,
      "elapsed_s": 8038.647321939468,
      "loss": 2.812240958213806,
      "hidden": 0.17376632243394852,
      "layer": 0.08603236451745033,
      "mlm": 1.4356873333454132,
      "distill": 0.9860048294067383,
      "teacher_acc": 0.7340800166130066
    },
    {
      "step": 10831,
      "lr": 1.2369426081357138e-05,
      "elapsed_s": 8046.051897764206,
      "loss": 2.9017006158828735,
      "hidden": 0.17419667169451714,
      "layer": 0.08619785495102406,
      "mlm": 1.5395678877830505,
      "distill": 0.9706405401229858,
      "teacher_acc": 0.7123187929391861
    },
    {
      "step": 10841,
      "lr": 1.2326003654660249e-05,
      "elapsed_s": 8053.444204568863,
      "loss": 2.926594853401184,
      "hidden": 0.17958706989884377,
      "layer": 0.08764453418552876,
      "mlm": 1.49676913022995,
      "distill": 1.0268293172121048,
      "teacher_acc": 0.7258793711662292
    },
    {
      "step": 10851,
      "lr": 1.2282632628065197e-05,
      "elapsed_s": 8060.847686052322,
      "loss": 3.173677980899811,
      "hidden": 0.1857832334935665,
      "layer": 0.09119569510221481,
      "mlm": 1.6747945249080658,
      "distill": 1.0817191302776337,
      "teacher_acc": 0.6949105560779572
    },
    {
      "step": 10861,
      "lr": 1.223931317746561e-05,
      "elapsed_s": 8068.2799236774445,
      "loss": 2.845583736896515,
      "hidden": 0.17506690695881844,
      "layer": 0.08639275655150414,
      "mlm": 1.5101032257080078,
      "distill": 0.942150354385376,
      "teacher_acc": 0.7145123034715652
    },
    {
      "step": 10871,
      "lr": 1.219604547854595e-05,
      "elapsed_s": 8075.700533151627,
      "loss": 3.042316496372223,
      "hidden": 0.18063481152057648,
      "layer": 0.08822735212743282,
      "mlm": 1.5936723053455353,
      "distill": 1.0432609468698502,
      "teacher_acc": 0.7096281051635742
    },
    {
      "step": 10881,
      "lr": 1.2152829706780786e-05,
      "elapsed_s": 8083.133833885193,
      "loss": 2.80331152677536,
      "hidden": 0.17149009928107262,
      "layer": 0.08549987524747849,
      "mlm": 1.50066077709198,
      "distill": 0.9169205576181412,
      "teacher_acc": 0.7157619893550873
    },
    {
      "step": 10891,
      "lr": 1.2109666037434103e-05,
      "elapsed_s": 8090.562560796738,
      "loss": 2.862439513206482,
      "hidden": 0.17378314211964607,
      "layer": 0.08636759035289288,
      "mlm": 1.4983938038349152,
      "distill": 0.9732956290245056,
      "teacher_acc": 0.7172624170780182
    },
    {
      "step": 10901,
      "lr": 1.2066554645558578e-05,
      "elapsed_s": 8098.022439241409,
      "loss": 2.923736035823822,
      "hidden": 0.17649774625897408,
      "layer": 0.08684984408318996,
      "mlm": 1.5769773125648499,
      "distill": 0.9503383189439774,
      "teacher_acc": 0.7077839225530624
    },
    {
      "step": 10911,
      "lr": 1.2023495705994878e-05,
      "elapsed_s": 8105.443075180054,
      "loss": 2.952065646648407,
      "hidden": 0.18032432720065117,
      "layer": 0.08837340399622917,
      "mlm": 1.5177892744541168,
      "distill": 1.0294411182403564,
      "teacher_acc": 0.7318726181983948
    },
    {
      "step": 10921,
      "lr": 1.1980489393370938e-05,
      "elapsed_s": 8112.8568115234375,
      "loss": 2.97021347284317,
      "hidden": 0.1823689453303814,
      "layer": 0.0889456607401371,
      "mlm": 1.5114178657531738,
      "distill": 1.0495848804712296,
      "teacher_acc": 0.7260873317718506
    },
    {
      "step": 10931,
      "lr": 1.1937535882101281e-05,
      "elapsed_s": 8120.26229095459,
      "loss": 2.8764527440071106,
      "hidden": 0.17667071893811226,
      "layer": 0.08732515946030617,
      "mlm": 1.4817224144935608,
      "distill": 0.9977263063192368,
      "teacher_acc": 0.7302416115999222
    },
    {
      "step": 10941,
      "lr": 1.1894635346386253e-05,
      "elapsed_s": 8127.668026685715,
      "loss": 2.897518754005432,
      "hidden": 0.1780134253203869,
      "layer": 0.08816490694880486,
      "mlm": 1.491661012172699,
      "distill": 1.0057484954595566,
      "teacher_acc": 0.733996793627739
    },
    {
      "step": 10951,
      "lr": 1.1851787960211398e-05,
      "elapsed_s": 8135.0835971832275,
      "loss": 3.082518696784973,
      "hidden": 0.18760431185364723,
      "layer": 0.091263297945261,
      "mlm": 1.5622581541538239,
      "distill": 1.0994202196598053,
      "teacher_acc": 0.7154341638088226
    },
    {
      "step": 10961,
      "lr": 1.180899389734668e-05,
      "elapsed_s": 8142.51713514328,
      "loss": 2.9126681685447693,
      "hidden": 0.17907466739416122,
      "layer": 0.08786855451762676,
      "mlm": 1.5165916085243225,
      "distill": 0.9939929097890854,
      "teacher_acc": 0.7216907143592834
    },
    {
      "step": 10971,
      "lr": 1.1766253331345819e-05,
      "elapsed_s": 8149.943874835968,
      "loss": 2.8558509349823,
      "hidden": 0.17591554298996925,
      "layer": 0.08646071888506413,
      "mlm": 1.5233791768550873,
      "distill": 0.9374103546142578,
      "teacher_acc": 0.714735209941864
    },
    {
      "step": 10981,
      "lr": 1.1723566435545586e-05,
      "elapsed_s": 8157.37261390686,
      "loss": 2.9577229022979736,
      "hidden": 0.18060832098126411,
      "layer": 0.0887327566742897,
      "mlm": 1.4503415524959564,
      "distill": 1.1017983257770538,
      "teacher_acc": 0.7396913170814514
    },
    {
      "step": 10991,
      "lr": 1.1680933383065057e-05,
      "elapsed_s": 8164.815044164658,
      "loss": 2.9793737530708313,
      "hidden": 0.18209616467356682,
      "layer": 0.08843136020004749,
      "mlm": 1.5104252696037292,
      "distill": 1.0605405569076538,
      "teacher_acc": 0.7222954481840134
    },
    {
      "step": 11001,
      "lr": 1.1638354346804971e-05,
      "elapsed_s": 8174.292570114136,
      "loss": 3.0591729879379272,
      "hidden": 0.18375826254487038,
      "layer": 0.08906274847686291,
      "mlm": 1.5488409101963043,
      "distill": 1.0982841551303864,
      "teacher_acc": 0.713960126042366
    },
    {
      "step": 11011,
      "lr": 1.1595829499446997e-05,
      "elapsed_s": 8181.700609445572,
      "loss": 2.811732530593872,
      "hidden": 0.17359792441129684,
      "layer": 0.0867298599332571,
      "mlm": 1.430332601070404,
      "distill": 0.9908391237258911,
      "teacher_acc": 0.7297311723232269
    },
    {
      "step": 11021,
      "lr": 1.1553359013453036e-05,
      "elapsed_s": 8189.092926263809,
      "loss": 2.8688098192214966,
      "hidden": 0.17574268206954002,
      "layer": 0.08731205761432648,
      "mlm": 1.5155298113822937,
      "distill": 0.9581385403871536,
      "teacher_acc": 0.718981072306633
    },
    {
      "step": 11031,
      "lr": 1.1510943061064524e-05,
      "elapsed_s": 8196.488725423813,
      "loss": 2.889613449573517,
      "hidden": 0.1785399131476879,
      "layer": 0.086368378251791,
      "mlm": 1.532465547323227,
      "distill": 0.9568838775157928,
      "teacher_acc": 0.7118146568536758
    },
    {
      "step": 11041,
      "lr": 1.1468581814301717e-05,
      "elapsed_s": 8203.896468400955,
      "loss": 2.8355020880699158,
      "hidden": 0.18041084706783295,
      "layer": 0.08836122043430805,
      "mlm": 1.39651557803154,
      "distill": 1.0339842736721039,
      "teacher_acc": 0.7444476783275604
    },
    {
      "step": 11051,
      "lr": 1.1426275444963034e-05,
      "elapsed_s": 8211.325413942337,
      "loss": 2.823788046836853,
      "hidden": 0.17146672680974007,
      "layer": 0.08550742268562317,
      "mlm": 1.4881187081336975,
      "distill": 0.9499821811914444,
      "teacher_acc": 0.7154545933008194
    },
    {
      "step": 11061,
      "lr": 1.1384024124624324e-05,
      "elapsed_s": 8218.712444782257,
      "loss": 3.0636661052703857,
      "hidden": 0.18050988391041756,
      "layer": 0.08853267505764961,
      "mlm": 1.5883564352989197,
      "distill": 1.07002355158329,
      "teacher_acc": 0.705975741147995
    },
    {
      "step": 11071,
      "lr": 1.1341828024638176e-05,
      "elapsed_s": 8226.138909816742,
      "loss": 2.8460269570350647,
      "hidden": 0.17565151304006577,
      "layer": 0.08687569014728069,
      "mlm": 1.4517238140106201,
      "distill": 0.9995622485876083,
      "teacher_acc": 0.7355851233005524
    },
    {
      "step": 11081,
      "lr": 1.1299687316133256e-05,
      "elapsed_s": 8233.527619123459,
      "loss": 2.897119104862213,
      "hidden": 0.18363207206130028,
      "layer": 0.09011726267635822,
      "mlm": 1.4538487195968628,
      "distill": 1.030947670340538,
      "teacher_acc": 0.7334865927696228
    },
    {
      "step": 11091,
      "lr": 1.125760217001354e-05,
      "elapsed_s": 8240.94069647789,
      "loss": 2.8547372817993164,
      "hidden": 0.18133947998285294,
      "layer": 0.0880105160176754,
      "mlm": 1.4419240951538086,
      "distill": 1.0061289370059967,
      "teacher_acc": 0.7333982437849045
    },
    {
      "step": 11101,
      "lr": 1.121557275695771e-05,
      "elapsed_s": 8248.357316493988,
      "loss": 2.8644628524780273,
      "hidden": 0.17004140466451645,
      "layer": 0.08553404547274113,
      "mlm": 1.4861394464969635,
      "distill": 0.9954736083745956,
      "teacher_acc": 0.7312144190073013
    },
    {
      "step": 11111,
      "lr": 1.1173599247418398e-05,
      "elapsed_s": 8255.764517068863,
      "loss": 2.893744111061096,
      "hidden": 0.17784874141216278,
      "layer": 0.08707399480044842,
      "mlm": 1.5358798801898956,
      "distill": 0.9586297124624252,
      "teacher_acc": 0.7142466902732849
    },
    {
      "step": 11121,
      "lr": 1.1131681811621527e-05,
      "elapsed_s": 8263.166898965836,
      "loss": 2.870635986328125,
      "hidden": 0.17893068492412567,
      "layer": 0.0879601389169693,
      "mlm": 1.50857412815094,
      "distill": 0.9602203816175461,
      "teacher_acc": 0.7221059054136276
    },
    {
      "step": 11131,
      "lr": 1.1089820619565613e-05,
      "elapsed_s": 8270.537461996078,
      "loss": 2.7392388582229614,
      "hidden": 0.17465965822339058,
      "layer": 0.08662197552621365,
      "mlm": 1.3773939311504364,
      "distill": 0.969214603304863,
      "teacher_acc": 0.7373160570859909
    },
    {
      "step": 11141,
      "lr": 1.1048015841021045e-05,
      "elapsed_s": 8277.955116271973,
      "loss": 2.9043917655944824,
      "hidden": 0.1735965758562088,
      "layer": 0.08871979638934135,
      "mlm": 1.535517692565918,
      "distill": 0.9773210138082504,
      "teacher_acc": 0.7159198969602585
    },
    {
      "step": 11151,
      "lr": 1.100626764552945e-05,
      "elapsed_s": 8285.359418392181,
      "loss": 2.7507399916648865,
      "hidden": 0.1750749908387661,
      "layer": 0.08621581457555294,
      "mlm": 1.3997158110141754,
      "distill": 0.9577663242816925,
      "teacher_acc": 0.7371083199977875
    },
    {
      "step": 11161,
      "lr": 1.096457620240298e-05,
      "elapsed_s": 8292.770998239517,
      "loss": 2.9044637084007263,
      "hidden": 0.17918264865875244,
      "layer": 0.08818322420120239,
      "mlm": 1.5083374083042145,
      "distill": 0.9936694204807281,
      "teacher_acc": 0.7113655060529709
    },
    {
      "step": 11171,
      "lr": 1.0922941680723609e-05,
      "elapsed_s": 8300.178480148315,
      "loss": 2.853361964225769,
      "hidden": 0.18020496144890785,
      "layer": 0.08840948529541492,
      "mlm": 1.4550070762634277,
      "distill": 0.9937402456998825,
      "teacher_acc": 0.7284946739673615
    },
    {
      "step": 11181,
      "lr": 1.0881364249342482e-05,
      "elapsed_s": 8307.562896251678,
      "loss": 2.9452579617500305,
      "hidden": 0.18045639619231224,
      "layer": 0.08787625655531883,
      "mlm": 1.539931297302246,
      "distill": 1.000475749373436,
      "teacher_acc": 0.7181874066591263
    },
    {
      "step": 11191,
      "lr": 1.0839844076879185e-05,
      "elapsed_s": 8314.93463420868,
      "loss": 3.0016876459121704,
      "hidden": 0.17725010961294174,
      "layer": 0.08656693622469902,
      "mlm": 1.5905376970767975,
      "distill": 1.0133661776781082,
      "teacher_acc": 0.7064286321401596
    },
    {
      "step": 11201,
      "lr": 1.0798381331721109e-05,
      "elapsed_s": 8322.33320736885,
      "loss": 2.978957772254944,
      "hidden": 0.1827639564871788,
      "layer": 0.08845875971019268,
      "mlm": 1.5099029839038849,
      "distill": 1.0592975169420242,
      "teacher_acc": 0.7345392107963562
    },
    {
      "step": 11211,
      "lr": 1.0756976182022738e-05,
      "elapsed_s": 8329.735498905182,
      "loss": 2.8994853496551514,
      "hidden": 0.18071771413087845,
      "layer": 0.08622293546795845,
      "mlm": 1.504266768693924,
      "distill": 0.9906716197729111,
      "teacher_acc": 0.7245052307844162
    },
    {
      "step": 11221,
      "lr": 1.0715628795704976e-05,
      "elapsed_s": 8337.119082450867,
      "loss": 2.89864045381546,
      "hidden": 0.17699796706438065,
      "layer": 0.08716765977442265,
      "mlm": 1.5176758170127869,
      "distill": 0.9833848476409912,
      "teacher_acc": 0.7159259617328644
    },
    {
      "step": 11231,
      "lr": 1.0674339340454475e-05,
      "elapsed_s": 8344.486906290054,
      "loss": 2.8088842630386353,
      "hidden": 0.17852935567498207,
      "layer": 0.0869959183037281,
      "mlm": 1.437552809715271,
      "distill": 0.9707747846841812,
      "teacher_acc": 0.7310845106840134
    },
    {
      "step": 11241,
      "lr": 1.0633107983722912e-05,
      "elapsed_s": 8351.880491256714,
      "loss": 2.8802369832992554,
      "hidden": 0.17473775893449783,
      "layer": 0.08646500110626221,
      "mlm": 1.4971394538879395,
      "distill": 0.9903894811868668,
      "teacher_acc": 0.7188897430896759
    },
    {
      "step": 11251,
      "lr": 1.059193489272637e-05,
      "elapsed_s": 8359.259387016296,
      "loss": 2.9411877393722534,
      "hidden": 0.17962602525949478,
      "layer": 0.08809495531022549,
      "mlm": 1.5223579108715057,
      "distill": 1.0155303329229355,
      "teacher_acc": 0.7106743305921555
    },
    {
      "step": 11261,
      "lr": 1.0550820234444627e-05,
      "elapsed_s": 8366.676477432251,
      "loss": 2.908635199069977,
      "hidden": 0.17515680193901062,
      "layer": 0.0872745718806982,
      "mlm": 1.5419666469097137,
      "distill": 0.972717672586441,
      "teacher_acc": 0.712832972407341
    },
    {
      "step": 11271,
      "lr": 1.0509764175620485e-05,
      "elapsed_s": 8374.091731071472,
      "loss": 2.8143728971481323,
      "hidden": 0.1776893511414528,
      "layer": 0.08701508119702339,
      "mlm": 1.4451144933700562,
      "distill": 0.9703721851110458,
      "teacher_acc": 0.7318149507045746
    },
    {
      "step": 11281,
      "lr": 1.0468766882759094e-05,
      "elapsed_s": 8381.511043548584,
      "loss": 2.824914813041687,
      "hidden": 0.17489682510495186,
      "layer": 0.08707134239375591,
      "mlm": 1.4373705387115479,
      "distill": 0.9942149072885513,
      "teacher_acc": 0.7397149503231049
    },
    {
      "step": 11291,
      "lr": 1.042782852212725e-05,
      "elapsed_s": 8388.949609518051,
      "loss": 2.738795578479767,
      "hidden": 0.17358281463384628,
      "layer": 0.08582963235676289,
      "mlm": 1.4490820169448853,
      "distill": 0.8996331244707108,
      "teacher_acc": 0.7238960415124893
    },
    {
      "step": 11301,
      "lr": 1.0386949259752785e-05,
      "elapsed_s": 8396.338478803635,
      "loss": 2.897427797317505,
      "hidden": 0.17370864376425743,
      "layer": 0.08691506274044514,
      "mlm": 1.5385419130325317,
      "distill": 0.9680110216140747,
      "teacher_acc": 0.7181727737188339
    },
    {
      "step": 11311,
      "lr": 1.0346129261423829e-05,
      "elapsed_s": 8403.761209964752,
      "loss": 2.827839434146881,
      "hidden": 0.1758677139878273,
      "layer": 0.08664453215897083,
      "mlm": 1.4731989800930023,
      "distill": 0.959582731127739,
      "teacher_acc": 0.7223649621009827
    },
    {
      "step": 11321,
      "lr": 1.0305368692688174e-05,
      "elapsed_s": 8411.167393922806,
      "loss": 2.8418572545051575,
      "hidden": 0.17750455811619759,
      "layer": 0.08730238117277622,
      "mlm": 1.541614055633545,
      "distill": 0.9015828520059586,
      "teacher_acc": 0.7091637849807739
    },
    {
      "step": 11331,
      "lr": 1.0264667718852596e-05,
      "elapsed_s": 8418.592934846878,
      "loss": 2.9483031034469604,
      "hidden": 0.18206826597452164,
      "layer": 0.08887264877557755,
      "mlm": 1.5284055471420288,
      "distill": 1.0113246589899063,
      "teacher_acc": 0.7131596952676773
    },
    {
      "step": 11341,
      "lr": 1.022402650498216e-05,
      "elapsed_s": 8426.022907018661,
      "loss": 2.825657546520233,
      "hidden": 0.17415465414524078,
      "layer": 0.08641180768609047,
      "mlm": 1.4823412895202637,
      "distill": 0.9518010765314102,
      "teacher_acc": 0.7236685901880264
    },
    {
      "step": 11351,
      "lr": 1.0183445215899584e-05,
      "elapsed_s": 8433.44666981697,
      "loss": 2.9158015847206116,
      "hidden": 0.17578015476465225,
      "layer": 0.08694989047944546,
      "mlm": 1.530271828174591,
      "distill": 0.9904945194721222,
      "teacher_acc": 0.7128785252571106
    },
    {
      "step": 11361,
      "lr": 1.0142924016184568e-05,
      "elapsed_s": 8440.859045505524,
      "loss": 2.898588180541992,
      "hidden": 0.17610974609851837,
      "layer": 0.08696969226002693,
      "mlm": 1.5033361315727234,
      "distill": 0.9995477795600891,
      "teacher_acc": 0.7262991666793823
    },
    {
      "step": 11371,
      "lr": 1.01024630701731e-05,
      "elapsed_s": 8448.274160861969,
      "loss": 3.0256532430648804,
      "hidden": 0.18234818428754807,
      "layer": 0.08838297985494137,
      "mlm": 1.5864404439926147,
      "distill": 1.0303249061107635,
      "teacher_acc": 0.7109770625829697
    },
    {
      "step": 11381,
      "lr": 1.0062062541956819e-05,
      "elapsed_s": 8455.704915761948,
      "loss": 2.896264433860779,
      "hidden": 0.1722925379872322,
      "layer": 0.08687197417020798,
      "mlm": 1.5256859958171844,
      "distill": 0.9825573712587357,
      "teacher_acc": 0.7164729684591293
    },
    {
      "step": 11391,
      "lr": 1.0021722595382324e-05,
      "elapsed_s": 8463.129249811172,
      "loss": 2.8619176149368286,
      "hidden": 0.17419498413801193,
      "layer": 0.08679201267659664,
      "mlm": 1.5226439237594604,
      "distill": 0.9474877566099167,
      "teacher_acc": 0.7074283063411713
    },
    {
      "step": 11401,
      "lr": 9.981443394050525e-06,
      "elapsed_s": 8470.574127674103,
      "loss": 2.8641727566719055,
      "hidden": 0.18065128102898598,
      "layer": 0.08835645206272602,
      "mlm": 1.4715473651885986,
      "distill": 0.9871447086334229,
      "teacher_acc": 0.7299947142601013
    },
    {
      "step": 11411,
      "lr": 9.941225101315985e-06,
      "elapsed_s": 8477.975200176239,
      "loss": 2.9649810194969177,
      "hidden": 0.18841061741113663,
      "layer": 0.09030756168067455,
      "mlm": 1.4516553282737732,
      "distill": 1.0913506895303726,
      "teacher_acc": 0.7407376915216446
    },
    {
      "step": 11421,
      "lr": 9.901067880286235e-06,
      "elapsed_s": 8485.38941192627,
      "loss": 2.80659556388855,
      "hidden": 0.17252138629555702,
      "layer": 0.08550152741372585,
      "mlm": 1.466357707977295,
      "distill": 0.9524443596601486,
      "teacher_acc": 0.7323374450206757
    },
    {
      "step": 11431,
      "lr": 9.860971893821146e-06,
      "elapsed_s": 8492.786566257477,
      "loss": 2.8184033632278442,
      "hidden": 0.18014365062117577,
      "layer": 0.08865422755479813,
      "mlm": 1.4193513095378876,
      "distill": 0.9944376349449158,
      "teacher_acc": 0.7395077347755432
    },
    {
      "step": 11441,
      "lr": 9.820937304532221e-06,
      "elapsed_s": 8500.177840948105,
      "loss": 2.9423749446868896,
      "hidden": 0.1759333796799183,
      "layer": 0.08721175417304039,
      "mlm": 1.5651275217533112,
      "distill": 0.9817748218774796,
      "teacher_acc": 0.7038186490535736
    },
    {
      "step": 11451,
      "lr": 9.780964274781984e-06,
      "elapsed_s": 8507.578917741776,
      "loss": 2.8845903277397156,
      "hidden": 0.18232486769557,
      "layer": 0.08678038604557514,
      "mlm": 1.4921681582927704,
      "distill": 0.9843822568655014,
      "teacher_acc": 0.7170978635549545
    },
    {
      "step": 11461,
      "lr": 9.741052966683294e-06,
      "elapsed_s": 8514.973740816116,
      "loss": 2.9620314836502075,
      "hidden": 0.17883280664682388,
      "layer": 0.08934056758880615,
      "mlm": 1.596371352672577,
      "distill": 0.9633242040872574,
      "teacher_acc": 0.689149022102356
    },
    {
      "step": 11471,
      "lr": 9.7012035420987e-06,
      "elapsed_s": 8522.418118715286,
      "loss": 2.944342076778412,
      "hidden": 0.1806718222796917,
      "layer": 0.08816816098988056,
      "mlm": 1.5170991718769073,
      "distill": 1.021815150976181,
      "teacher_acc": 0.718651607632637
    },
    {
      "step": 11481,
      "lr": 9.66141616263978e-06,
      "elapsed_s": 8529.84995007515,
      "loss": 2.8999286890029907,
      "hidden": 0.17628328502178192,
      "layer": 0.0872431006282568,
      "mlm": 1.5417075753211975,
      "distill": 0.9620329290628433,
      "teacher_acc": 0.7076239287853241
    },
    {
      "step": 11491,
      "lr": 9.621690989666466e-06,
      "elapsed_s": 8537.30105805397,
      "loss": 2.8464741110801697,
      "hidden": 0.17579087242484093,
      "layer": 0.08706557750701904,
      "mlm": 1.497264564037323,
      "distill": 0.9540950208902359,
      "teacher_acc": 0.7276084423065186
    },
    {
      "step": 11501,
      "lr": 9.582028184286423e-06,
      "elapsed_s": 8544.708785533905,
      "loss": 2.885443389415741,
      "hidden": 0.17321577668190002,
      "layer": 0.08608797565102577,
      "mlm": 1.5416736900806427,
      "distill": 0.9542940706014633,
      "teacher_acc": 0.7116115689277649
    },
    {
      "step": 11511,
      "lr": 9.542427907354388e-06,
      "elapsed_s": 8552.149512529373,
      "loss": 2.885970413684845,
      "hidden": 0.18107197806239128,
      "layer": 0.08855819143354893,
      "mlm": 1.4717864990234375,
      "distill": 1.0077608972787857,
      "teacher_acc": 0.7346422225236893
    },
    {
      "step": 11521,
      "lr": 9.502890319471491e-06,
      "elapsed_s": 8559.52800488472,
      "loss": 2.9210673570632935,
      "hidden": 0.18180108815431595,
      "layer": 0.08815604075789452,
      "mlm": 1.5517668724060059,
      "distill": 0.9616202712059021,
      "teacher_acc": 0.7162619680166245
    },
    {
      "step": 11531,
      "lr": 9.463415580984653e-06,
      "elapsed_s": 8566.934325695038,
      "loss": 2.8077496886253357,
      "hidden": 0.18027520552277565,
      "layer": 0.08763755857944489,
      "mlm": 1.4142753779888153,
      "distill": 0.9891051352024078,
      "teacher_acc": 0.7354015558958054
    },
    {
      "step": 11541,
      "lr": 9.42400385198586e-06,
      "elapsed_s": 8574.341815948486,
      "loss": 2.980990469455719,
      "hidden": 0.180645439773798,
      "layer": 0.08775981143116951,
      "mlm": 1.562819391489029,
      "distill": 1.013000339269638,
      "teacher_acc": 0.7137101590633392
    },
    {
      "step": 11551,
      "lr": 9.3846552923116e-06,
      "elapsed_s": 8581.730875730515,
      "loss": 2.897901773452759,
      "hidden": 0.17938899621367455,
      "layer": 0.08682218566536903,
      "mlm": 1.479920893907547,
      "distill": 1.0157918184995651,
      "teacher_acc": 0.7351734191179276
    },
    {
      "step": 11561,
      "lr": 9.345370061542158e-06,
      "elapsed_s": 8589.13337278366,
      "loss": 2.8499958515167236,
      "hidden": 0.1732211969792843,
      "layer": 0.08609162271022797,
      "mlm": 1.5467858016490936,
      "distill": 0.9137218296527863,
      "teacher_acc": 0.7115947306156158
    },
    {
      "step": 11571,
      "lr": 9.306148319000987e-06,
      "elapsed_s": 8596.513417005539,
      "loss": 2.8371639251708984,
      "hidden": 0.18070733919739723,
      "layer": 0.0883625652641058,
      "mlm": 1.4371312856674194,
      "distill": 0.994436651468277,
      "teacher_acc": 0.7410618811845779
    },
    {
      "step": 11581,
      "lr": 9.266990223754069e-06,
      "elapsed_s": 8603.903351783752,
      "loss": 2.8276089429855347,
      "hidden": 0.17636990174651146,
      "layer": 0.08723104745149612,
      "mlm": 1.4409880936145782,
      "distill": 0.9902655184268951,
      "teacher_acc": 0.7365210205316544
    },
    {
      "step": 11591,
      "lr": 9.227895934609235e-06,
      "elapsed_s": 8611.298266172409,
      "loss": 2.908479630947113,
      "hidden": 0.17557913064956665,
      "layer": 0.0872409250587225,
      "mlm": 1.5281242430210114,
      "distill": 0.9855766892433167,
      "teacher_acc": 0.7225566357374191
    },
    {
      "step": 11601,
      "lr": 9.18886561011557e-06,
      "elapsed_s": 8618.642398118973,
      "loss": 2.913455545902252,
      "hidden": 0.18657585605978966,
      "layer": 0.08977381512522697,
      "mlm": 1.458300918340683,
      "distill": 1.0371160060167313,
      "teacher_acc": 0.7322098165750504
    },
    {
      "step": 11611,
      "lr": 9.149899408562745e-06,
      "elapsed_s": 8626.027275562286,
      "loss": 2.7774588465690613,
      "hidden": 0.1750325784087181,
      "layer": 0.0866832323372364,
      "mlm": 1.4090064764022827,
      "distill": 0.9750455617904663,
      "teacher_acc": 0.7393578290939331
    },
    {
      "step": 11621,
      "lr": 9.110997487980368e-06,
      "elapsed_s": 8633.433954954147,
      "loss": 3.0530285239219666,
      "hidden": 0.1826467029750347,
      "layer": 0.08954072929918766,
      "mlm": 1.6053223311901093,
      "distill": 1.0376424342393875,
      "teacher_acc": 0.7058679312467575
    },
    {
      "step": 11631,
      "lr": 9.072160006137367e-06,
      "elapsed_s": 8640.843310117722,
      "loss": 3.0000651478767395,
      "hidden": 0.18736637011170387,
      "layer": 0.09122733399271965,
      "mlm": 1.5146294832229614,
      "distill": 1.0650892555713654,
      "teacher_acc": 0.7262876927852631
    },
    {
      "step": 11641,
      "lr": 9.033387120541306e-06,
      "elapsed_s": 8648.24194765091,
      "loss": 2.9398823380470276,
      "hidden": 0.18013345822691917,
      "layer": 0.08839108608663082,
      "mlm": 1.5299261808395386,
      "distill": 1.0054936856031418,
      "teacher_acc": 0.7185913473367691
    },
    {
      "step": 11651,
      "lr": 8.994678988437802e-06,
      "elapsed_s": 8655.653980731964,
      "loss": 2.758953332901001,
      "hidden": 0.1731620542705059,
      "layer": 0.08695606701076031,
      "mlm": 1.4302538335323334,
      "distill": 0.9388974159955978,
      "teacher_acc": 0.731830433011055
    },
    {
      "step": 11661,
      "lr": 8.956035766809853e-06,
      "elapsed_s": 8663.048542499542,
      "loss": 2.961130976676941,
      "hidden": 0.16989745944738388,
      "layer": 0.08577794022858143,
      "mlm": 1.5996421575546265,
      "distill": 0.9788049161434174,
      "teacher_acc": 0.7035162150859833
    },
    {
      "step": 11671,
      "lr": 8.917457612377209e-06,
      "elapsed_s": 8670.456528425217,
      "loss": 2.8528011441230774,
      "hidden": 0.17657870799303055,
      "layer": 0.0878971554338932,
      "mlm": 1.4519985020160675,
      "distill": 1.0036966800689697,
      "teacher_acc": 0.7303527295589447
    },
    {
      "step": 11681,
      "lr": 8.878944681595742e-06,
      "elapsed_s": 8677.858333349228,
      "loss": 2.968489646911621,
      "hidden": 0.17904430255293846,
      "layer": 0.08768092468380928,
      "mlm": 1.573696345090866,
      "distill": 0.992864266037941,
      "teacher_acc": 0.7049909383058548
    },
    {
      "step": 11691,
      "lr": 8.840497130656783e-06,
      "elapsed_s": 8685.288291454315,
      "loss": 2.863508641719818,
      "hidden": 0.17423254996538162,
      "layer": 0.08627472631633282,
      "mlm": 1.5310141146183014,
      "distill": 0.9408920556306839,
      "teacher_acc": 0.7106742560863495
    },
    {
      "step": 11701,
      "lr": 8.802115115486535e-06,
      "elapsed_s": 8692.70529294014,
      "loss": 2.824951171875,
      "hidden": 0.1755516491830349,
      "layer": 0.08589500561356544,
      "mlm": 1.4995906054973602,
      "distill": 0.9313097149133682,
      "teacher_acc": 0.7210783660411835
    },
    {
      "step": 11711,
      "lr": 8.763798791745411e-06,
      "elapsed_s": 8700.105291843414,
      "loss": 3.013614237308502,
      "hidden": 0.1790935955941677,
      "layer": 0.08957492746412754,
      "mlm": 1.5755402743816376,
      "distill": 1.0350993275642395,
      "teacher_acc": 0.7133761942386627
    },
    {
      "step": 11721,
      "lr": 8.725548314827406e-06,
      "elapsed_s": 8707.488520860672,
      "loss": 2.8423375487327576,
      "hidden": 0.1754515841603279,
      "layer": 0.08722830563783646,
      "mlm": 1.4775493741035461,
      "distill": 0.9702708274126053,
      "teacher_acc": 0.7252146899700165
    },
    {
      "step": 11731,
      "lr": 8.687363839859482e-06,
      "elapsed_s": 8714.86055970192,
      "loss": 2.8149014711380005,
      "hidden": 0.1809549257159233,
      "layer": 0.08803625218570232,
      "mlm": 1.3803109228610992,
      "distill": 1.0286626368761063,
      "teacher_acc": 0.7506801635026932
    },
    {
      "step": 11741,
      "lr": 8.649245521700897e-06,
      "elapsed_s": 8722.239639043808,
      "loss": 2.9604023694992065,
      "hidden": 0.17346380650997162,
      "layer": 0.08687496185302734,
      "mlm": 1.574812412261963,
      "distill": 0.9952247738838196,
      "teacher_acc": 0.7058846205472946
    },
    {
      "step": 11751,
      "lr": 8.611193514942634e-06,
      "elapsed_s": 8729.628894329071,
      "loss": 2.7676438689231873,
      "hidden": 0.17304710671305656,
      "layer": 0.08634073659777641,
      "mlm": 1.4414955377578735,
      "distill": 0.936883807182312,
      "teacher_acc": 0.7326495349407196
    },
    {
      "step": 11761,
      "lr": 8.573207973906735e-06,
      "elapsed_s": 8737.024418592453,
      "loss": 2.859658181667328,
      "hidden": 0.17218875139951706,
      "layer": 0.08656189776957035,
      "mlm": 1.5494816601276398,
      "distill": 0.9225181192159653,
      "teacher_acc": 0.7119984179735184
    },
    {
      "step": 11771,
      "lr": 8.535289052645689e-06,
      "elapsed_s": 8744.437524080276,
      "loss": 2.8856375217437744,
      "hidden": 0.17475572228431702,
      "layer": 0.08731837943196297,
      "mlm": 1.5041763186454773,
      "distill": 0.988290548324585,
      "teacher_acc": 0.7194439768791199
    },
    {
      "step": 11781,
      "lr": 8.497436904941813e-06,
      "elapsed_s": 8751.860045194626,
      "loss": 2.8977131247520447,
      "hidden": 0.1758020892739296,
      "layer": 0.08620345592498779,
      "mlm": 1.5046611726284027,
      "distill": 0.9983460754156113,
      "teacher_acc": 0.729612410068512
    },
    {
      "step": 11791,
      "lr": 8.459651684306597e-06,
      "elapsed_s": 8759.314459085464,
      "loss": 2.850575864315033,
      "hidden": 0.17216972261667252,
      "layer": 0.0855779591947794,
      "mlm": 1.4903175830841064,
      "distill": 0.9731298983097076,
      "teacher_acc": 0.7278861254453659
    },
    {
      "step": 11801,
      "lr": 8.421933543980126e-06,
      "elapsed_s": 8766.745078086853,
      "loss": 3.096903443336487,
      "hidden": 0.19534436613321304,
      "layer": 0.0924385953694582,
      "mlm": 1.4864759743213654,
      "distill": 1.1735194325447083,
      "teacher_acc": 0.7453214079141617
    },
    {
      "step": 11811,
      "lr": 8.384282636930427e-06,
      "elapsed_s": 8774.165004253387,
      "loss": 2.933634400367737,
      "hidden": 0.1803545467555523,
      "layer": 0.08867593854665756,
      "mlm": 1.5361057817935944,
      "distill": 0.9924814701080322,
      "teacher_acc": 0.7087429314851761
    },
    {
      "step": 11821,
      "lr": 8.346699115852865e-06,
      "elapsed_s": 8781.567938804626,
      "loss": 3.028900682926178,
      "hidden": 0.184206061065197,
      "layer": 0.08979523368179798,
      "mlm": 1.5382551550865173,
      "distill": 1.0773358047008514,
      "teacher_acc": 0.7254036962985992
    },
    {
      "step": 11831,
      "lr": 8.309183133169524e-06,
      "elapsed_s": 8788.969747066498,
      "loss": 2.920081853866577,
      "hidden": 0.18216409161686897,
      "layer": 0.088149668648839,
      "mlm": 1.4883165657520294,
      "distill": 1.0233622789382935,
      "teacher_acc": 0.7370486557483673
    },
    {
      "step": 11841,
      "lr": 8.271734841028553e-06,
      "elapsed_s": 8796.383830308914,
      "loss": 2.9474685192108154,
      "hidden": 0.17692579701542854,
      "layer": 0.08774906024336815,
      "mlm": 1.5228377878665924,
      "distill": 1.026904582977295,
      "teacher_acc": 0.7267642915248871
    },
    {
      "step": 11851,
      "lr": 8.234354391303605e-06,
      "elapsed_s": 8803.8024725914,
      "loss": 2.88394296169281,
      "hidden": 0.17380856350064278,
      "layer": 0.08699492551386356,
      "mlm": 1.4950258135795593,
      "distill": 0.9978025406599045,
      "teacher_acc": 0.7335617244243622
    },
    {
      "step": 11861,
      "lr": 8.197041935593181e-06,
      "elapsed_s": 8811.217730045319,
      "loss": 2.957947313785553,
      "hidden": 0.17782976850867271,
      "layer": 0.08744676597416401,
      "mlm": 1.5441320836544037,
      "distill": 1.0144322961568832,
      "teacher_acc": 0.721091166138649
    },
    {
      "step": 11871,
      "lr": 8.159797625220058e-06,
      "elapsed_s": 8818.603947162628,
      "loss": 2.8275174498558044,
      "hidden": 0.17556288093328476,
      "layer": 0.08728029765188694,
      "mlm": 1.4604116678237915,
      "distill": 0.9723398983478546,
      "teacher_acc": 0.729839101433754
    },
    {
      "step": 11881,
      "lr": 8.1226216112306e-06,
      "elapsed_s": 8825.980924844742,
      "loss": 2.872560143470764,
      "hidden": 0.1737397015094757,
      "layer": 0.08614519238471985,
      "mlm": 1.5144849121570587,
      "distill": 0.9675232470035553,
      "teacher_acc": 0.7148987948894501
    },
    {
      "step": 11891,
      "lr": 8.085514044394216e-06,
      "elapsed_s": 8833.368203639984,
      "loss": 2.7765116691589355,
      "hidden": 0.172513660043478,
      "layer": 0.08582797460258007,
      "mlm": 1.4176904261112213,
      "distill": 0.9708799421787262,
      "teacher_acc": 0.7234673351049423
    },
    {
      "step": 11901,
      "lr": 8.048475075202727e-06,
      "elapsed_s": 8840.742236614227,
      "loss": 2.947446286678314,
      "hidden": 0.17943421751260757,
      "layer": 0.08848432637751102,
      "mlm": 1.5572600066661835,
      "distill": 0.987075611948967,
      "teacher_acc": 0.7124486416578293
    },
    {
      "step": 11911,
      "lr": 8.011504853869742e-06,
      "elapsed_s": 8848.141516923904,
      "loss": 2.838305115699768,
      "hidden": 0.17934516444802284,
      "layer": 0.08803470432758331,
      "mlm": 1.4406099915504456,
      "distill": 0.9949873834848404,
      "teacher_acc": 0.7388346493244171
    },
    {
      "step": 11921,
      "lr": 7.974603530330069e-06,
      "elapsed_s": 8855.52515912056,
      "loss": 2.8661672472953796,
      "hidden": 0.17183957621455193,
      "layer": 0.08613789640367031,
      "mlm": 1.4858387410640717,
      "distill": 0.9935803562402725,
      "teacher_acc": 0.7249466776847839
    },
    {
      "step": 11931,
      "lr": 7.937771254239093e-06,
      "elapsed_s": 8862.926328897476,
      "loss": 2.8321884870529175,
      "hidden": 0.17948558181524277,
      "layer": 0.08656339161098003,
      "mlm": 1.4419536292552948,
      "distill": 0.9879820197820663,
      "teacher_acc": 0.7307125777006149
    },
    {
      "step": 11941,
      "lr": 7.90100817497216e-06,
      "elapsed_s": 8870.328876972198,
      "loss": 2.878299295902252,
      "hidden": 0.17104396224021912,
      "layer": 0.0856066457927227,
      "mlm": 1.573552131652832,
      "distill": 0.9198558777570724,
      "teacher_acc": 0.703888401389122
    },
    {
      "step": 11951,
      "lr": 7.864314441624004e-06,
      "elapsed_s": 8877.737349033356,
      "loss": 2.9189100861549377,
      "hidden": 0.17678238078951836,
      "layer": 0.08745857700705528,
      "mlm": 1.5330724716186523,
      "distill": 0.9885434657335281,
      "teacher_acc": 0.7160025238990784
    },
    {
      "step": 11961,
      "lr": 7.827690203008118e-06,
      "elapsed_s": 8885.143232584,
      "loss": 2.75415700674057,
      "hidden": 0.1705382652580738,
      "layer": 0.08538345992565155,
      "mlm": 1.4551694989204407,
      "distill": 0.9152192771434784,
      "teacher_acc": 0.7301317602396011
    },
    {
      "step": 11971,
      "lr": 7.791135607656147e-06,
      "elapsed_s": 8892.542583703995,
      "loss": 2.7730290293693542,
      "hidden": 0.17468402534723282,
      "layer": 0.08606037497520447,
      "mlm": 1.4452482461929321,
      "distill": 0.9353825002908707,
      "teacher_acc": 0.7274285107851028
    },
    {
      "step": 11981,
      "lr": 7.75465080381731e-06,
      "elapsed_s": 8899.964803695679,
      "loss": 2.847308933734894,
      "hidden": 0.17436283081769943,
      "layer": 0.08659091964364052,
      "mlm": 1.4638864994049072,
      "distill": 0.991401344537735,
      "teacher_acc": 0.7308118939399719
    },
    {
      "step": 11991,
      "lr": 7.718235939457758e-06,
      "elapsed_s": 8907.36946773529,
      "loss": 2.8775338530540466,
      "hidden": 0.18212231621146202,
      "layer": 0.0885999258607626,
      "mlm": 1.4232809245586395,
      "distill": 1.04570834338665,
      "teacher_acc": 0.7512505948543549
    },
    {
      "step": 12001,
      "lr": 7.681891162260015e-06,
      "elapsed_s": 8916.832163095474,
      "loss": 2.9314300417900085,
      "hidden": 0.17291953787207603,
      "layer": 0.08634726144373417,
      "mlm": 1.6006021201610565,
      "distill": 0.9418151527643204,
      "teacher_acc": 0.7041100710630417
    },
    {
      "step": 12011,
      "lr": 7.645616619622364e-06,
      "elapsed_s": 8924.242282152176,
      "loss": 2.834304451942444,
      "hidden": 0.1728130467236042,
      "layer": 0.0858449824154377,
      "mlm": 1.5005516409873962,
      "distill": 0.9452042132616043,
      "teacher_acc": 0.7179669886827469
    },
    {
      "step": 12021,
      "lr": 7.609412458658244e-06,
      "elapsed_s": 8931.641526222229,
      "loss": 2.8811034560203552,
      "hidden": 0.17153335362672806,
      "layer": 0.0857680905610323,
      "mlm": 1.511580765247345,
      "distill": 0.9835719615221024,
      "teacher_acc": 0.7131068408489227
    },
    {
      "step": 12031,
      "lr": 7.573278826195665e-06,
      "elapsed_s": 8939.061381816864,
      "loss": 2.835046112537384,
      "hidden": 0.1780453883111477,
      "layer": 0.08725152164697647,
      "mlm": 1.4468506872653961,
      "distill": 0.9884789288043976,
      "teacher_acc": 0.7248119562864304
    },
    {
      "step": 12041,
      "lr": 7.5372158687765784e-06,
      "elapsed_s": 8946.45587682724,
      "loss": 2.9230287671089172,
      "hidden": 0.17428746446967125,
      "layer": 0.08651290461421013,
      "mlm": 1.5514414608478546,
      "distill": 0.9797559529542923,
      "teacher_acc": 0.7113645821809769
    },
    {
      "step": 12051,
      "lr": 7.5012237326563374e-06,
      "elapsed_s": 8953.857497215271,
      "loss": 2.862743377685547,
      "hidden": 0.176184494048357,
      "layer": 0.08719086274504662,
      "mlm": 1.4713284373283386,
      "distill": 0.9954505115747452,
      "teacher_acc": 0.7348389327526093
    },
    {
      "step": 12061,
      "lr": 7.465302563803061e-06,
      "elapsed_s": 8961.270748853683,
      "loss": 2.842897951602936,
      "hidden": 0.1768861785531044,
      "layer": 0.08717388659715652,
      "mlm": 1.4549515545368195,
      "distill": 0.990587055683136,
      "teacher_acc": 0.7331886440515518
    },
    {
      "step": 12071,
      "lr": 7.429452507897069e-06,
      "elapsed_s": 8968.68651175499,
      "loss": 2.9320555925369263,
      "hidden": 0.18492050096392632,
      "layer": 0.08943217433989048,
      "mlm": 1.4415404498577118,
      "distill": 1.0759581625461578,
      "teacher_acc": 0.7384877651929855
    },
    {
      "step": 12081,
      "lr": 7.393673710330271e-06,
      "elapsed_s": 8976.08201622963,
      "loss": 2.8461391925811768,
      "hidden": 0.1724015325307846,
      "layer": 0.08482075482606888,
      "mlm": 1.5250435173511505,
      "distill": 0.9338822662830353,
      "teacher_acc": 0.7225817292928696
    },
    {
      "step": 12091,
      "lr": 7.357966316205572e-06,
      "elapsed_s": 8983.494993448257,
      "loss": 2.982856333255768,
      "hidden": 0.17760394513607025,
      "layer": 0.08684525638818741,
      "mlm": 1.5729098618030548,
      "distill": 1.0113159865140915,
      "teacher_acc": 0.7036267518997192
    },
    {
      "step": 12101,
      "lr": 7.3223304703363135e-06,
      "elapsed_s": 8990.903176307678,
      "loss": 2.875020444393158,
      "hidden": 0.1775338537991047,
      "layer": 0.08723405562341213,
      "mlm": 1.4764533340930939,
      "distill": 0.9998824000358582,
      "teacher_acc": 0.7245829701423645
    },
    {
      "step": 12111,
      "lr": 7.286766317245667e-06,
      "elapsed_s": 8998.329807281494,
      "loss": 2.947964072227478,
      "hidden": 0.18002808839082718,
      "layer": 0.08798137679696083,
      "mlm": 1.5169947445392609,
      "distill": 1.02692249417305,
      "teacher_acc": 0.7321084439754486
    },
    {
      "step": 12121,
      "lr": 7.251274001166044e-06,
      "elapsed_s": 9005.743683099747,
      "loss": 2.938950836658478,
      "hidden": 0.17796747013926506,
      "layer": 0.0879037082195282,
      "mlm": 1.5837144255638123,
      "distill": 0.9553495645523071,
      "teacher_acc": 0.6981218308210373
    },
    {
      "step": 12131,
      "lr": 7.21585366603853e-06,
      "elapsed_s": 9013.143601417542,
      "loss": 2.9448598623275757,
      "hidden": 0.17359739914536476,
      "layer": 0.08700956404209137,
      "mlm": 1.5934978127479553,
      "distill": 0.9606625139713287,
      "teacher_acc": 0.7057676315307617
    },
    {
      "step": 12141,
      "lr": 7.180505455512257e-06,
      "elapsed_s": 9020.588706493378,
      "loss": 2.8078258633613586,
      "hidden": 0.1770188920199871,
      "layer": 0.08722705021500587,
      "mlm": 1.462302029132843,
      "distill": 0.9478724598884583,
      "teacher_acc": 0.7265336066484451
    },
    {
      "step": 12151,
      "lr": 7.145229512943885e-06,
      "elapsed_s": 9027.999309062958,
      "loss": 3.004558503627777,
      "hidden": 0.18746376410126686,
      "layer": 0.0897368025034666,
      "mlm": 1.5376700162887573,
      "distill": 1.0470924973487854,
      "teacher_acc": 0.7244391739368439
    },
    {
      "step": 12161,
      "lr": 7.110025981396975e-06,
      "elapsed_s": 9035.426665782928,
      "loss": 2.8593517541885376,
      "hidden": 0.1728067249059677,
      "layer": 0.08650297485291958,
      "mlm": 1.4706211388111115,
      "distill": 0.9998656660318375,
      "teacher_acc": 0.7279855757951736
    },
    {
      "step": 12171,
      "lr": 7.074895003641416e-06,
      "elapsed_s": 9042.856859922409,
      "loss": 2.83821439743042,
      "hidden": 0.17724983021616936,
      "layer": 0.08762476406991482,
      "mlm": 1.451403021812439,
      "distill": 0.9884994179010391,
      "teacher_acc": 0.7327395230531693
    },
    {
      "step": 12181,
      "lr": 7.039836722152865e-06,
      "elapsed_s": 9050.303092479706,
      "loss": 2.8467034101486206,
      "hidden": 0.1751006618142128,
      "layer": 0.08681318536400795,
      "mlm": 1.510590821504593,
      "distill": 0.9425046592950821,
      "teacher_acc": 0.718340739607811
    },
    {
      "step": 12191,
      "lr": 7.004851279112129e-06,
      "elapsed_s": 9057.720226049423,
      "loss": 2.742767333984375,
      "hidden": 0.17188889160752296,
      "layer": 0.0852587316185236,
      "mlm": 1.4319615066051483,
      "distill": 0.9243986904621124,
      "teacher_acc": 0.7241169810295105
    },
    {
      "step": 12201,
      "lr": 6.969938816404639e-06,
      "elapsed_s": 9065.113187789917,
      "loss": 2.8497883081436157,
      "hidden": 0.17528287321329117,
      "layer": 0.08592304028570652,
      "mlm": 1.4684549570083618,
      "distill": 0.9878060668706894,
      "teacher_acc": 0.7345440685749054
    },
    {
      "step": 12211,
      "lr": 6.935099475619847e-06,
      "elapsed_s": 9072.514890909195,
      "loss": 2.939872086048126,
      "hidden": 0.1748601570725441,
      "layer": 0.08751777373254299,
      "mlm": 1.5013472735881805,
      "distill": 1.0450455844402313,
      "teacher_acc": 0.7260120660066605
    },
    {
      "step": 12221,
      "lr": 6.900333398050646e-06,
      "elapsed_s": 9079.923726320267,
      "loss": 3.0436598658561707,
      "hidden": 0.18855173885822296,
      "layer": 0.09064742922782898,
      "mlm": 1.4749193489551544,
      "distill": 1.1463133841753006,
      "teacher_acc": 0.742340162396431
    },
    {
      "step": 12231,
      "lr": 6.865640724692815e-06,
      "elapsed_s": 9087.297847509384,
      "loss": 2.8712212443351746,
      "hidden": 0.17112068459391594,
      "layer": 0.085650984197855,
      "mlm": 1.5542872846126556,
      "distill": 0.9318671673536301,
      "teacher_acc": 0.7158327698707581
    },
    {
      "step": 12241,
      "lr": 6.831021596244424e-06,
      "elapsed_s": 9094.621049880981,
      "loss": 2.967933714389801,
      "hidden": 0.17635205388069153,
      "layer": 0.0875365175306797,
      "mlm": 1.5603772401809692,
      "distill": 1.0110841542482376,
      "teacher_acc": 0.7126067578792572
    },
    {
      "step": 12251,
      "lr": 6.796476153105294e-06,
      "elapsed_s": 9101.958236694336,
      "loss": 2.8774508237838745,
      "hidden": 0.176167793571949,
      "layer": 0.08700981736183167,
      "mlm": 1.5261945128440857,
      "distill": 0.9554157704114914,
      "teacher_acc": 0.7251121550798416
    },
    {
      "step": 12261,
      "lr": 6.762004535376401e-06,
      "elapsed_s": 9109.346936225891,
      "loss": 2.9645684957504272,
      "hidden": 0.1791466623544693,
      "layer": 0.08887865953147411,
      "mlm": 1.5504562854766846,
      "distill": 1.0113795101642609,
      "teacher_acc": 0.7152739763259888
    },
    {
      "step": 12271,
      "lr": 6.727606882859322e-06,
      "elapsed_s": 9116.738011360168,
      "loss": 2.7975228428840637,
      "hidden": 0.1711507961153984,
      "layer": 0.08498072065412998,
      "mlm": 1.4801384508609772,
      "distill": 0.9325924664735794,
      "teacher_acc": 0.7187763154506683
    },
    {
      "step": 12281,
      "lr": 6.69328333505567e-06,
      "elapsed_s": 9124.153195142746,
      "loss": 2.9614241123199463,
      "hidden": 0.17723563686013222,
      "layer": 0.08690509386360645,
      "mlm": 1.5811430215835571,
      "distill": 0.9823572337627411,
      "teacher_acc": 0.7007973194122314
    },
    {
      "step": 12291,
      "lr": 6.6590340311665e-06,
      "elapsed_s": 9131.575122594833,
      "loss": 3.0274060368537903,
      "hidden": 0.18156101554632187,
      "layer": 0.08857851661741734,
      "mlm": 1.5915491878986359,
      "distill": 1.0284454971551895,
      "teacher_acc": 0.711734876036644
    },
    {
      "step": 12301,
      "lr": 6.624859110091791e-06,
      "elapsed_s": 9138.995480775833,
      "loss": 2.8635968565940857,
      "hidden": 0.1792665384709835,
      "layer": 0.08671696297824383,
      "mlm": 1.468196302652359,
      "distill": 0.9935090243816376,
      "teacher_acc": 0.7233945578336716
    },
    {
      "step": 12311,
      "lr": 6.590758710429848e-06,
      "elapsed_s": 9146.3963804245,
      "loss": 2.995170831680298,
      "hidden": 0.17937440797686577,
      "layer": 0.08788389898836613,
      "mlm": 1.5653658211231232,
      "distill": 1.02711421251297,
      "teacher_acc": 0.7209800034761429
    },
    {
      "step": 12321,
      "lr": 6.556732970476748e-06,
      "elapsed_s": 9153.803459405899,
      "loss": 2.9733137488365173,
      "hidden": 0.18749452009797096,
      "layer": 0.09024551883339882,
      "mlm": 1.4758122563362122,
      "distill": 1.0773897469043732,
      "teacher_acc": 0.7330247759819031
    },
    {
      "step": 12331,
      "lr": 6.522782028225788e-06,
      "elapsed_s": 9161.204004764557,
      "loss": 3.018292248249054,
      "hidden": 0.18321746587753296,
      "layer": 0.08959094993770123,
      "mlm": 1.5303759574890137,
      "distill": 1.076685905456543,
      "teacher_acc": 0.7263926267623901
    },
    {
      "step": 12341,
      "lr": 6.488906021366925e-06,
      "elapsed_s": 9168.619151115417,
      "loss": 2.900115728378296,
      "hidden": 0.17203755304217339,
      "layer": 0.08626106381416321,
      "mlm": 1.5114847719669342,
      "distill": 1.0014252811670303,
      "teacher_acc": 0.7154399901628494
    },
    {
      "step": 12351,
      "lr": 6.455105087286173e-06,
      "elapsed_s": 9176.030782699585,
      "loss": 2.8845927119255066,
      "hidden": 0.176792673766613,
      "layer": 0.0880075041204691,
      "mlm": 1.5009724497795105,
      "distill": 0.9860312044620514,
      "teacher_acc": 0.7257981896400452
    },
    {
      "step": 12361,
      "lr": 6.421379363065142e-06,
      "elapsed_s": 9183.446513414383,
      "loss": 2.9109652638435364,
      "hidden": 0.18113511055707932,
      "layer": 0.08939162828028202,
      "mlm": 1.4997273981571198,
      "distill": 1.0042718648910522,
      "teacher_acc": 0.7226878702640533
    },
    {
      "step": 12371,
      "lr": 6.3877289854803815e-06,
      "elapsed_s": 9190.869278907776,
      "loss": 2.925171375274658,
      "hidden": 0.1764317937195301,
      "layer": 0.08810182847082615,
      "mlm": 1.5235716700553894,
      "distill": 1.0046852827072144,
      "teacher_acc": 0.7173303812742233
    },
    {
      "step": 12381,
      "lr": 6.354154091002892e-06,
      "elapsed_s": 9198.286696195602,
      "loss": 2.940699517726898,
      "hidden": 0.17727218195796013,
      "layer": 0.08717207238078117,
      "mlm": 1.540239155292511,
      "distill": 1.00232994556427,
      "teacher_acc": 0.7188484072685242
    },
    {
      "step": 12391,
      "lr": 6.320654815797525e-06,
      "elapsed_s": 9205.657669067383,
      "loss": 2.87903094291687,
      "hidden": 0.17641252279281616,
      "layer": 0.08733411878347397,
      "mlm": 1.4878909289836884,
      "distill": 0.9946479201316833,
      "teacher_acc": 0.725951075553894
    },
    {
      "step": 12401,
      "lr": 6.28723129572247e-06,
      "elapsed_s": 9213.067886590958,
      "loss": 2.8834495544433594,
      "hidden": 0.17526870965957642,
      "layer": 0.08773728460073471,
      "mlm": 1.5381772816181183,
      "distill": 0.9508662223815918,
      "teacher_acc": 0.7072023004293442
    },
    {
      "step": 12411,
      "lr": 6.253883666328689e-06,
      "elapsed_s": 9220.45101594925,
      "loss": 2.81054550409317,
      "hidden": 0.17354321852326393,
      "layer": 0.08565921895205975,
      "mlm": 1.4706654846668243,
      "distill": 0.9499640017747879,
      "teacher_acc": 0.7152236849069595
    },
    {
      "step": 12421,
      "lr": 6.220612062859357e-06,
      "elapsed_s": 9227.855909824371,
      "loss": 2.9474292397499084,
      "hidden": 0.18893872573971748,
      "layer": 0.09117171913385391,
      "mlm": 1.4707243144512177,
      "distill": 1.0532415956258774,
      "teacher_acc": 0.7262109816074371
    },
    {
      "step": 12431,
      "lr": 6.187416620249334e-06,
      "elapsed_s": 9235.299842357635,
      "loss": 2.8519203662872314,
      "hidden": 0.18094110488891602,
      "layer": 0.08845470659434795,
      "mlm": 1.4203514456748962,
      "distill": 1.025459423661232,
      "teacher_acc": 0.7377538979053497
    },
    {
      "step": 12441,
      "lr": 6.154297473124582e-06,
      "elapsed_s": 9242.705707550049,
      "loss": 2.919790804386139,
      "hidden": 0.18504753336310387,
      "layer": 0.0893565695732832,
      "mlm": 1.4471072554588318,
      "distill": 1.0579102635383606,
      "teacher_acc": 0.7390169054269791
    },
    {
      "step": 12451,
      "lr": 6.121254755801659e-06,
      "elapsed_s": 9250.111196994781,
      "loss": 2.970368981361389,
      "hidden": 0.18696963787078857,
      "layer": 0.08923973143100739,
      "mlm": 1.469307541847229,
      "distill": 1.0825023651123047,
      "teacher_acc": 0.7354277670383453
    },
    {
      "step": 12461,
      "lr": 6.088288602287159e-06,
      "elapsed_s": 9257.537653446198,
      "loss": 2.984572172164917,
      "hidden": 0.18187499046325684,
      "layer": 0.08817940019071102,
      "mlm": 1.5212276875972748,
      "distill": 1.055504858493805,
      "teacher_acc": 0.7203629910945892
    },
    {
      "step": 12471,
      "lr": 6.055399146277166e-06,
      "elapsed_s": 9264.983309984207,
      "loss": 2.7818097472190857,
      "hidden": 0.17320609465241432,
      "layer": 0.08638905361294746,
      "mlm": 1.4695047438144684,
      "distill": 0.9226982146501541,
      "teacher_acc": 0.7247221022844315
    },
    {
      "step": 12481,
      "lr": 6.022586521156715e-06,
      "elapsed_s": 9272.37607216835,
      "loss": 2.8497734665870667,
      "hidden": 0.17899274826049805,
      "layer": 0.08826789632439613,
      "mlm": 1.4984366595745087,
      "distill": 0.949217364192009,
      "teacher_acc": 0.7178781777620316
    },
    {
      "step": 12491,
      "lr": 5.989850859999227e-06,
      "elapsed_s": 9279.776977300644,
      "loss": 2.9110100269317627,
      "hidden": 0.17424943298101425,
      "layer": 0.08676040731370449,
      "mlm": 1.5412298440933228,
      "distill": 0.977901041507721,
      "teacher_acc": 0.7151438444852829
    },
    {
      "step": 12501,
      "lr": 5.957192295566022e-06,
      "elapsed_s": 9287.164252281189,
      "loss": 2.7939191460609436,
      "hidden": 0.1739400289952755,
      "layer": 0.08644329383969307,
      "mlm": 1.4476470947265625,
      "distill": 0.9551702588796616,
      "teacher_acc": 0.7361476868391037
    },
    {
      "step": 12511,
      "lr": 5.924610960305732e-06,
      "elapsed_s": 9294.519112586975,
      "loss": 2.84879332780838,
      "hidden": 0.1788582168519497,
      "layer": 0.08684664033353329,
      "mlm": 1.4495508074760437,
      "distill": 0.9981028139591217,
      "teacher_acc": 0.7454253882169724
    },
    {
      "step": 12521,
      "lr": 5.89210698635379e-06,
      "elapsed_s": 9301.901784658432,
      "loss": 2.8524584770202637,
      "hidden": 0.17203010246157646,
      "layer": 0.08625171519815922,
      "mlm": 1.52052640914917,
      "distill": 0.9447459876537323,
      "teacher_acc": 0.7128268927335739
    },
    {
      "step": 12531,
      "lr": 5.859680505531887e-06,
      "elapsed_s": 9309.305381059647,
      "loss": 2.9700721502304077,
      "hidden": 0.1778218299150467,
      "layer": 0.0875339675694704,
      "mlm": 1.560851514339447,
      "distill": 1.009809985756874,
      "teacher_acc": 0.7061426639556885
    },
    {
      "step": 12541,
      "lr": 5.827331649347417e-06,
      "elapsed_s": 9316.677250385284,
      "loss": 3.0179476141929626,
      "hidden": 0.1761799454689026,
      "layer": 0.08757508173584938,
      "mlm": 1.6000151932239532,
      "distill": 1.021785020828247,
      "teacher_acc": 0.7089514434337616
    },
    {
      "step": 12551,
      "lr": 5.79506054899299e-06,
      "elapsed_s": 9324.051854610443,
      "loss": 2.7452880144119263,
      "hidden": 0.17222030460834503,
      "layer": 0.0865737572312355,
      "mlm": 1.4091834127902985,
      "distill": 0.9483770430088043,
      "teacher_acc": 0.7394954413175583
    },
    {
      "step": 12561,
      "lr": 5.7628673353458504e-06,
      "elapsed_s": 9331.41999411583,
      "loss": 2.7567885518074036,
      "hidden": 0.17702651768922806,
      "layer": 0.08583145588636398,
      "mlm": 1.429551512002945,
      "distill": 0.9302682876586914,
      "teacher_acc": 0.740069791674614
    },
    {
      "step": 12571,
      "lr": 5.730752138967388e-06,
      "elapsed_s": 9338.82175397873,
      "loss": 2.7882784605026245,
      "hidden": 0.17645251750946045,
      "layer": 0.08597725071012974,
      "mlm": 1.4847504496574402,
      "distill": 0.9076343029737473,
      "teacher_acc": 0.7077653259038925
    },
    {
      "step": 12581,
      "lr": 5.6987150901025855e-06,
      "elapsed_s": 9346.205089092255,
      "loss": 3.0471176505088806,
      "hidden": 0.1886911503970623,
      "layer": 0.09075606428086758,
      "mlm": 1.498786747455597,
      "distill": 1.1255705803632736,
      "teacher_acc": 0.7388933300971985
    },
    {
      "step": 12591,
      "lr": 5.66675631867947e-06,
      "elapsed_s": 9353.59905052185,
      "loss": 2.8596156239509583,
      "hidden": 0.18239294737577438,
      "layer": 0.08967356942594051,
      "mlm": 1.3950499594211578,
      "distill": 1.0549429804086685,
      "teacher_acc": 0.7583006322383881
    },
    {
      "step": 12601,
      "lr": 5.634875954308638e-06,
      "elapsed_s": 9360.993413686752,
      "loss": 2.9558114409446716,
      "hidden": 0.17997289448976517,
      "layer": 0.08749737218022346,
      "mlm": 1.5507054924964905,
      "distill": 1.0014115124940872,
      "teacher_acc": 0.717472717165947
    },
    {
      "step": 12611,
      "lr": 5.603074126282692e-06,
      "elapsed_s": 9368.385633468628,
      "loss": 2.860435903072357,
      "hidden": 0.17809220403432846,
      "layer": 0.08761430904269218,
      "mlm": 1.492460459470749,
      "distill": 0.9679839015007019,
      "teacher_acc": 0.7297491878271103
    },
    {
      "step": 12621,
      "lr": 5.571350963575728e-06,
      "elapsed_s": 9375.78344154358,
      "loss": 2.753363311290741,
      "hidden": 0.17181523144245148,
      "layer": 0.08667032979428768,
      "mlm": 1.449986845254898,
      "distill": 0.9164108484983444,
      "teacher_acc": 0.7283483594655991
    },
    {
      "step": 12631,
      "lr": 5.539706594842814e-06,
      "elapsed_s": 9383.168736934662,
      "loss": 2.8144888877868652,
      "hidden": 0.1860583834350109,
      "layer": 0.08957562409341335,
      "mlm": 1.3462290167808533,
      "distill": 1.0513553470373154,
      "teacher_acc": 0.7634536772966385
    },
    {
      "step": 12641,
      "lr": 5.5081411484194435e-06,
      "elapsed_s": 9390.579170942307,
      "loss": 2.7688790559768677,
      "hidden": 0.1732962615787983,
      "layer": 0.08614427037537098,
      "mlm": 1.4611330032348633,
      "distill": 0.9180813580751419,
      "teacher_acc": 0.7259592115879059
    },
    {
      "step": 12651,
      "lr": 5.476654752321064e-06,
      "elapsed_s": 9397.98660326004,
      "loss": 2.8267762660980225,
      "hidden": 0.18169112130999565,
      "layer": 0.08811956644058228,
      "mlm": 1.4484617710113525,
      "distill": 0.9708724170923233,
      "teacher_acc": 0.7284526526927948
    },
    {
      "step": 12661,
      "lr": 5.445247534242512e-06,
      "elapsed_s": 9405.392384052277,
      "loss": 2.8898401856422424,
      "hidden": 0.17887293547391891,
      "layer": 0.08689091727137566,
      "mlm": 1.4635529816150665,
      "distill": 1.0250958502292633,
      "teacher_acc": 0.7397952824831009
    },
    {
      "step": 12671,
      "lr": 5.413919621557523e-06,
      "elapsed_s": 9412.784528255463,
      "loss": 2.918522357940674,
      "hidden": 0.17639200016856194,
      "layer": 0.08690697140991688,
      "mlm": 1.5518242418766022,
      "distill": 0.9704606533050537,
      "teacher_acc": 0.7114774733781815
    },
    {
      "step": 12681,
      "lr": 5.382671141318208e-06,
      "elapsed_s": 9420.215723276138,
      "loss": 2.8063570857048035,
      "hidden": 0.17836078628897667,
      "layer": 0.08761309273540974,
      "mlm": 1.4379312098026276,
      "distill": 0.9678977429866791,
      "teacher_acc": 0.7288535088300705
    },
    {
      "step": 12691,
      "lr": 5.351502220254512e-06,
      "elapsed_s": 9427.617034435272,
      "loss": 2.9242215752601624,
      "hidden": 0.1845530904829502,
      "layer": 0.08981956169009209,
      "mlm": 1.4409829676151276,
      "distill": 1.0692225992679596,
      "teacher_acc": 0.7366450428962708
    },
    {
      "step": 12701,
      "lr": 5.320412984773748e-06,
      "elapsed_s": 9435.031553268433,
      "loss": 2.9122524857521057,
      "hidden": 0.17955318465828896,
      "layer": 0.08945824019610882,
      "mlm": 1.4768843352794647,
      "distill": 1.0315327495336533,
      "teacher_acc": 0.7308914810419083
    },
    {
      "step": 12711,
      "lr": 5.289403560960052e-06,
      "elapsed_s": 9442.44949388504,
      "loss": 3.1997989416122437,
      "hidden": 0.18525556102395058,
      "layer": 0.0897974744439125,
      "mlm": 1.6119783222675323,
      "distill": 1.1724107265472412,
      "teacher_acc": 0.7179546654224396
    },
    {
      "step": 12721,
      "lr": 5.258474074573877e-06,
      "elapsed_s": 9449.87076997757,
      "loss": 2.887511909008026,
      "hidden": 0.17830026894807816,
      "layer": 0.08710087463259697,
      "mlm": 1.5254767835140228,
      "distill": 0.9618841111660004,
      "teacher_acc": 0.7233778536319733
    },
    {
      "step": 12731,
      "lr": 5.2276246510514965e-06,
      "elapsed_s": 9457.294798612595,
      "loss": 2.9076144099235535,
      "hidden": 0.17850354313850403,
      "layer": 0.0884072333574295,
      "mlm": 1.5025262534618378,
      "distill": 1.003877505660057,
      "teacher_acc": 0.7190152406692505
    },
    {
      "step": 12741,
      "lr": 5.196855415504456e-06,
      "elapsed_s": 9464.737800598145,
      "loss": 2.932520270347595,
      "hidden": 0.17750247940421104,
      "layer": 0.0877245832234621,
      "mlm": 1.5422592759132385,
      "distill": 0.9913936853408813,
      "teacher_acc": 0.7105195820331573
    },
    {
      "step": 12751,
      "lr": 5.166166492719124e-06,
      "elapsed_s": 9472.132479667664,
      "loss": 2.8253825306892395,
      "hidden": 0.1761576309800148,
      "layer": 0.08738777413964272,
      "mlm": 1.4342655539512634,
      "distill": 0.9951078146696091,
      "teacher_acc": 0.7354120910167694
    },
    {
      "step": 12761,
      "lr": 5.1355580071561465e-06,
      "elapsed_s": 9479.545963525772,
      "loss": 3.0643064975738525,
      "hidden": 0.19268574193120003,
      "layer": 0.09170088917016983,
      "mlm": 1.47756889462471,
      "distill": 1.1555155813694,
      "teacher_acc": 0.7403616458177567
    },
    {
      "step": 12771,
      "lr": 5.1050300829499465e-06,
      "elapsed_s": 9486.959055662155,
      "loss": 3.0280851125717163,
      "hidden": 0.1866171807050705,
      "layer": 0.09035687893629074,
      "mlm": 1.518099069595337,
      "distill": 1.0915732234716415,
      "teacher_acc": 0.7338429391384125
    },
    {
      "step": 12781,
      "lr": 5.074582843908241e-06,
      "elapsed_s": 9494.351012945175,
      "loss": 2.9281920790672302,
      "hidden": 0.18735918402671814,
      "layer": 0.09046633914113045,
      "mlm": 1.4711315035820007,
      "distill": 1.0371090769767761,
      "teacher_acc": 0.7321868538856506
    },
    {
      "step": 12791,
      "lr": 5.044216413511494e-06,
      "elapsed_s": 9501.750063419342,
      "loss": 2.867982804775238,
      "hidden": 0.18388090655207634,
      "layer": 0.0895646046847105,
      "mlm": 1.403470754623413,
      "distill": 1.0519679486751556,
      "teacher_acc": 0.7465981394052505
    },
    {
      "step": 12801,
      "lr": 5.013930914912476e-06,
      "elapsed_s": 9509.129306554794,
      "loss": 2.7903255820274353,
      "hidden": 0.1703214906156063,
      "layer": 0.08565936423838139,
      "mlm": 1.4558477699756622,
      "distill": 0.9510051310062408,
      "teacher_acc": 0.7316219806671143
    },
    {
      "step": 12811,
      "lr": 4.983726470935721e-06,
      "elapsed_s": 9516.527590990067,
      "loss": 2.82758891582489,
      "hidden": 0.17570395022630692,
      "layer": 0.08738142624497414,
      "mlm": 1.4487531185150146,
      "distill": 0.9837372153997421,
      "teacher_acc": 0.7278356999158859
    },
    {
      "step": 12821,
      "lr": 4.95360320407704e-06,
      "elapsed_s": 9523.954644918442,
      "loss": 2.897589325904846,
      "hidden": 0.17975018918514252,
      "layer": 0.08901461400091648,
      "mlm": 1.4897750914096832,
      "distill": 1.0038065761327744,
      "teacher_acc": 0.7333808839321136
    },
    {
      "step": 12831,
      "lr": 4.923561236503033e-06,
      "elapsed_s": 9531.344733715057,
      "loss": 2.992402195930481,
      "hidden": 0.18500671163201332,
      "layer": 0.08914694748818874,
      "mlm": 1.474857211112976,
      "distill": 1.1029581427574158,
      "teacher_acc": 0.7388559579849243
    },
    {
      "step": 12841,
      "lr": 4.893600690050579e-06,
      "elapsed_s": 9538.727134227753,
      "loss": 2.890056610107422,
      "hidden": 0.17591312155127525,
      "layer": 0.08798208460211754,
      "mlm": 1.4926875531673431,
      "distill": 1.001551792025566,
      "teacher_acc": 0.7278572171926498
    },
    {
      "step": 12851,
      "lr": 4.86372168622635e-06,
      "elapsed_s": 9546.096660852432,
      "loss": 2.887464702129364,
      "hidden": 0.17923159524798393,
      "layer": 0.0877752099186182,
      "mlm": 1.4746856093406677,
      "distill": 1.0104282796382904,
      "teacher_acc": 0.7294062972068787
    },
    {
      "step": 12861,
      "lr": 4.833924346206322e-06,
      "elapsed_s": 9553.47875213623,
      "loss": 2.976298153400421,
      "hidden": 0.1795457825064659,
      "layer": 0.08712825924158096,
      "mlm": 1.5287494361400604,
      "distill": 1.0448929965496063,
      "teacher_acc": 0.7243291139602661
    },
    {
      "step": 12871,
      "lr": 4.8042087908352766e-06,
      "elapsed_s": 9560.872932434082,
      "loss": 2.7780619263648987,
      "hidden": 0.17690759524703026,
      "layer": 0.08809008821845055,
      "mlm": 1.4777454733848572,
      "distill": 0.902456134557724,
      "teacher_acc": 0.7153702229261398
    },
    {
      "step": 12881,
      "lr": 4.7745751406263165e-06,
      "elapsed_s": 9568.296875,
      "loss": 2.8947187066078186,
      "hidden": 0.17641475051641464,
      "layer": 0.08663193508982658,
      "mlm": 1.5103759467601776,
      "distill": 0.9881973415613174,
      "teacher_acc": 0.722339317202568
    },
    {
      "step": 12891,
      "lr": 4.745023515760358e-06,
      "elapsed_s": 9575.687087535858,
      "loss": 2.84686017036438,
      "hidden": 0.1764562986791134,
      "layer": 0.08673552051186562,
      "mlm": 1.4734503328800201,
      "distill": 0.9771295487880707,
      "teacher_acc": 0.7256836891174316
    },
    {
      "step": 12901,
      "lr": 4.715554036085673e-06,
      "elapsed_s": 9583.101128578186,
      "loss": 2.9557703733444214,
      "hidden": 0.18196866288781166,
      "layer": 0.08980824425816536,
      "mlm": 1.5450664460659027,
      "distill": 1.0018624812364578,
      "teacher_acc": 0.7099316865205765
    },
    {
      "step": 12911,
      "lr": 4.686166821117394e-06,
      "elapsed_s": 9590.482494354248,
      "loss": 3.006062626838684,
      "hidden": 0.17656185105443,
      "layer": 0.08685310557484627,
      "mlm": 1.622754991054535,
      "distill": 0.98675736784935,
      "teacher_acc": 0.7029636651277542
    },
    {
      "step": 12921,
      "lr": 4.656861990037009e-06,
      "elapsed_s": 9597.917001962662,
      "loss": 2.894753336906433,
      "hidden": 0.17806433141231537,
      "layer": 0.08680600300431252,
      "mlm": 1.5117867290973663,
      "distill": 0.9834349602460861,
      "teacher_acc": 0.7155244201421738
    },
    {
      "step": 12931,
      "lr": 4.6276396616919126e-06,
      "elapsed_s": 9605.31804561615,
      "loss": 2.960543155670166,
      "hidden": 0.17846061289310455,
      "layer": 0.08825407549738884,
      "mlm": 1.5462091565132141,
      "distill": 1.0132857710123062,
      "teacher_acc": 0.7113262414932251
    },
    {
      "step": 12941,
      "lr": 4.598499954594881e-06,
      "elapsed_s": 9612.73967218399,
      "loss": 2.956188440322876,
      "hidden": 0.1883348636329174,
      "layer": 0.09045879356563091,
      "mlm": 1.4664727449417114,
      "distill": 1.0678165704011917,
      "teacher_acc": 0.7251737415790558
    },
    {
      "step": 12951,
      "lr": 4.569442986923631e-06,
      "elapsed_s": 9620.14065527916,
      "loss": 3.0433897972106934,
      "hidden": 0.18353575468063354,
      "layer": 0.08933710679411888,
      "mlm": 1.5309925973415375,
      "distill": 1.1006570756435394,
      "teacher_acc": 0.7181217819452286
    },
    {
      "step": 12961,
      "lr": 4.540468876520323e-06,
      "elapsed_s": 9627.579767942429,
      "loss": 2.845133602619171,
      "hidden": 0.18036900833249092,
      "layer": 0.08745582588016987,
      "mlm": 1.4484224915504456,
      "distill": 0.992245152592659,
      "teacher_acc": 0.7332130074501038
    },
    {
      "step": 12971,
      "lr": 4.511577740891087e-06,
      "elapsed_s": 9635.008594751358,
      "loss": 2.917749583721161,
      "hidden": 0.17559747025370598,
      "layer": 0.08771784789860249,
      "mlm": 1.5513452589511871,
      "distill": 0.9713504463434219,
      "teacher_acc": 0.7102102935314178
    },
    {
      "step": 12981,
      "lr": 4.482769697205544e-06,
      "elapsed_s": 9642.426382780075,
      "loss": 2.878574550151825,
      "hidden": 0.17659876868128777,
      "layer": 0.08731736429035664,
      "mlm": 1.5271839201450348,
      "distill": 0.9545344263315201,
      "teacher_acc": 0.7203634530305862
    },
    {
      "step": 12991,
      "lr": 4.45404486229632e-06,
      "elapsed_s": 9649.839520215988,
      "loss": 3.045623540878296,
      "hidden": 0.18662891164422035,
      "layer": 0.08938478492200375,
      "mlm": 1.5411394536495209,
      "distill": 1.0865338742733002,
      "teacher_acc": 0.7275781482458115
    },
    {
      "step": 13001,
      "lr": 4.425403352658591e-06,
      "elapsed_s": 9659.282498598099,
      "loss": 2.9143919348716736,
      "hidden": 0.17616290971636772,
      "layer": 0.08753799460828304,
      "mlm": 1.531820297241211,
      "distill": 0.9864768981933594,
      "teacher_acc": 0.7170365452766418
    },
    {
      "step": 13011,
      "lr": 4.396845284449608e-06,
      "elapsed_s": 9666.668555498123,
      "loss": 2.8680944442749023,
      "hidden": 0.17702513933181763,
      "layer": 0.08729600347578526,
      "mlm": 1.5223488807678223,
      "distill": 0.9480472356081009,
      "teacher_acc": 0.717745378613472
    },
    {
      "step": 13021,
      "lr": 4.368370773488215e-06,
      "elapsed_s": 9674.033586740494,
      "loss": 3.015802502632141,
      "hidden": 0.18039514124393463,
      "layer": 0.08782284148037434,
      "mlm": 1.5655745267868042,
      "distill": 1.0455263257026672,
      "teacher_acc": 0.7257400006055832
    },
    {
      "step": 13031,
      "lr": 4.33997993525439e-06,
      "elapsed_s": 9681.417303085327,
      "loss": 2.931185007095337,
      "hidden": 0.1752033531665802,
      "layer": 0.08625833876430988,
      "mlm": 1.5731877386569977,
      "distill": 0.9644613862037659,
      "teacher_acc": 0.7013953626155853
    },
    {
      "step": 13041,
      "lr": 4.311672884888757e-06,
      "elapsed_s": 9688.817976474762,
      "loss": 2.870202124118805,
      "hidden": 0.17414305731654167,
      "layer": 0.08606310002505779,
      "mlm": 1.5003638863563538,
      "distill": 0.9785206615924835,
      "teacher_acc": 0.7138620018959045
    },
    {
      "step": 13051,
      "lr": 4.283449737192149e-06,
      "elapsed_s": 9696.208606004715,
      "loss": 2.9122373461723328,
      "hidden": 0.17695403844118118,
      "layer": 0.08789647929370403,
      "mlm": 1.5349477529525757,
      "distill": 0.9794332981109619,
      "teacher_acc": 0.7109359204769135
    },
    {
      "step": 13061,
      "lr": 4.255310606625123e-06,
      "elapsed_s": 9703.587857961655,
      "loss": 2.789665102958679,
      "hidden": 0.17151342704892159,
      "layer": 0.08601438626646996,
      "mlm": 1.4816088378429413,
      "distill": 0.9220222234725952,
      "teacher_acc": 0.7161831557750702
    },
    {
      "step": 13071,
      "lr": 4.227255607307496e-06,
      "elapsed_s": 9710.950668096542,
      "loss": 3.0212015509605408,
      "hidden": 0.17730635032057762,
      "layer": 0.08799059130251408,
      "mlm": 1.6220548748970032,
      "distill": 1.000538632273674,
      "teacher_acc": 0.6978080868721008
    },
    {
      "step": 13081,
      "lr": 4.199284853017896e-06,
      "elapsed_s": 9718.335723638535,
      "loss": 2.940508723258972,
      "hidden": 0.17891962081193924,
      "layer": 0.0870448611676693,
      "mlm": 1.5334879457950592,
      "distill": 1.0056591629981995,
      "teacher_acc": 0.7115296721458435
    },
    {
      "step": 13091,
      "lr": 4.171398457193274e-06,
      "elapsed_s": 9725.733839035034,
      "loss": 2.8743136525154114,
      "hidden": 0.176511000841856,
      "layer": 0.08644475601613522,
      "mlm": 1.5035784542560577,
      "distill": 0.9744907915592194,
      "teacher_acc": 0.7271473705768585
    },
    {
      "step": 13101,
      "lr": 4.143596532928468e-06,
      "elapsed_s": 9733.140990018845,
      "loss": 2.775389313697815,
      "hidden": 0.17423643171787262,
      "layer": 0.08601083233952522,
      "mlm": 1.4004228711128235,
      "distill": 0.9834882020950317,
      "teacher_acc": 0.7396484166383743
    },
    {
      "step": 13111,
      "lr": 4.115879192975747e-06,
      "elapsed_s": 9740.561436653137,
      "loss": 2.942090630531311,
      "hidden": 0.17801467329263687,
      "layer": 0.08715808764100075,
      "mlm": 1.5175164937973022,
      "distill": 1.0249657779932022,
      "teacher_acc": 0.7206477373838425
    },
    {
      "step": 13121,
      "lr": 4.088246549744331e-06,
      "elapsed_s": 9747.975591182709,
      "loss": 2.866785228252411,
      "hidden": 0.1745559647679329,
      "layer": 0.08631335012614727,
      "mlm": 1.498161256313324,
      "distill": 0.9763553291559219,
      "teacher_acc": 0.7243615686893463
    },
    {
      "step": 13131,
      "lr": 4.06069871529996e-06,
      "elapsed_s": 9755.345501422882,
      "loss": 2.925742983818054,
      "hidden": 0.1795111522078514,
      "layer": 0.08888725563883781,
      "mlm": 1.5192387402057648,
      "distill": 1.0030383616685867,
      "teacher_acc": 0.7144933193922043
    },
    {
      "step": 13141,
      "lr": 4.0332358013644016e-06,
      "elapsed_s": 9762.751586675644,
      "loss": 3.1206456422805786,
      "hidden": 0.17992790788412094,
      "layer": 0.08864620141685009,
      "mlm": 1.6399891674518585,
      "distill": 1.0764775723218918,
      "teacher_acc": 0.7002445757389069
    },
    {
      "step": 13151,
      "lr": 4.0058579193150535e-06,
      "elapsed_s": 9770.171698331833,
      "loss": 2.919557273387909,
      "hidden": 0.1816549077630043,
      "layer": 0.08843573741614819,
      "mlm": 1.4833087027072906,
      "distill": 1.0287209153175354,
      "teacher_acc": 0.7371143698692322
    },
    {
      "step": 13161,
      "lr": 3.978565180184449e-06,
      "elapsed_s": 9777.585371732712,
      "loss": 2.9087109565734863,
      "hidden": 0.18073051050305367,
      "layer": 0.08879253268241882,
      "mlm": 1.4650443196296692,
      "distill": 1.0378094017505646,
      "teacher_acc": 0.7353436499834061
    },
    {
      "step": 13171,
      "lr": 3.951357694659816e-06,
      "elapsed_s": 9784.985332489014,
      "loss": 2.9291704297065735,
      "hidden": 0.178382508456707,
      "layer": 0.08855052851140499,
      "mlm": 1.5377902686595917,
      "distill": 0.99033984541893,
      "teacher_acc": 0.7157203108072281
    },
    {
      "step": 13181,
      "lr": 3.924235573082644e-06,
      "elapsed_s": 9792.393367052078,
      "loss": 2.8498963713645935,
      "hidden": 0.17427730560302734,
      "layer": 0.08572972193360329,
      "mlm": 1.499645859003067,
      "distill": 0.9588310420513153,
      "teacher_acc": 0.7292221039533615
    },
    {
      "step": 13191,
      "lr": 3.897198925448201e-06,
      "elapsed_s": 9799.778712272644,
      "loss": 2.946622371673584,
      "hidden": 0.17694006115198135,
      "layer": 0.08735962398350239,
      "mlm": 1.5220724046230316,
      "distill": 1.0269900113344193,
      "teacher_acc": 0.7237818539142609
    },
    {
      "step": 13201,
      "lr": 3.8702478614051355e-06,
      "elapsed_s": 9807.189387321472,
      "loss": 2.7607191801071167,
      "hidden": 0.16602330654859543,
      "layer": 0.08407696336507797,
      "mlm": 1.5061639547348022,
      "distill": 0.8804701119661331,
      "teacher_acc": 0.714382603764534
    },
    {
      "step": 13211,
      "lr": 3.8433824902549935e-06,
      "elapsed_s": 9814.612425088882,
      "loss": 2.7988585829734802,
      "hidden": 0.17149867117404938,
      "layer": 0.08663110621273518,
      "mlm": 1.475572019815445,
      "distill": 0.9369736164808273,
      "teacher_acc": 0.7234218716621399
    },
    {
      "step": 13221,
      "lr": 3.816602920951792e-06,
      "elapsed_s": 9822.024948120117,
      "loss": 2.888017237186432,
      "hidden": 0.17855221033096313,
      "layer": 0.08737533539533615,
      "mlm": 1.4902073740959167,
      "distill": 0.9970177859067917,
      "teacher_acc": 0.7223703563213348
    },
    {
      "step": 13231,
      "lr": 3.789909262101582e-06,
      "elapsed_s": 9829.445343494415,
      "loss": 2.850633919239044,
      "hidden": 0.17753764614462852,
      "layer": 0.08807964622974396,
      "mlm": 1.4667156636714935,
      "distill": 0.9848031401634216,
      "teacher_acc": 0.737755075097084
    },
    {
      "step": 13241,
      "lr": 3.7633016219619786e-06,
      "elapsed_s": 9836.847332715988,
      "loss": 2.8578527569770813,
      "hidden": 0.17737197503447533,
      "layer": 0.08721950091421604,
      "mlm": 1.4773236215114594,
      "distill": 0.9821754395961761,
      "teacher_acc": 0.7243688851594925
    },
    {
      "step": 13251,
      "lr": 3.7367801084417618e-06,
      "elapsed_s": 9844.261668205261,
      "loss": 2.949710726737976,
      "hidden": 0.17219267785549164,
      "layer": 0.08628544770181179,
      "mlm": 1.6224125921726227,
      "distill": 0.9397700130939484,
      "teacher_acc": 0.691694587469101
    },
    {
      "step": 13261,
      "lr": 3.7103448291004176e-06,
      "elapsed_s": 9851.656147003174,
      "loss": 2.8338574171066284,
      "hidden": 0.17426765337586403,
      "layer": 0.08557852730154991,
      "mlm": 1.4594774842262268,
      "distill": 0.9830553233623505,
      "teacher_acc": 0.7253110706806183
    },
    {
      "step": 13271,
      "lr": 3.6839958911476957e-06,
      "elapsed_s": 9859.016288280487,
      "loss": 2.920965552330017,
      "hidden": 0.17562898248434067,
      "layer": 0.08750224113464355,
      "mlm": 1.493486374616623,
      "distill": 1.032470017671585,
      "teacher_acc": 0.7272103875875473
    },
    {
      "step": 13281,
      "lr": 3.6577334014432003e-06,
      "elapsed_s": 9866.402028083801,
      "loss": 3.1315943002700806,
      "hidden": 0.18815917521715164,
      "layer": 0.09127160534262657,
      "mlm": 1.603565126657486,
      "distill": 1.1060749888420105,
      "teacher_acc": 0.7184097319841385
    },
    {
      "step": 13291,
      "lr": 3.6315574664959164e-06,
      "elapsed_s": 9873.783749580383,
      "loss": 2.9850247502326965,
      "hidden": 0.17591681703925133,
      "layer": 0.0877597127109766,
      "mlm": 1.5690572559833527,
      "distill": 1.0202539265155792,
      "teacher_acc": 0.7122976183891296
    },
    {
      "step": 13301,
      "lr": 3.605468192463815e-06,
      "elapsed_s": 9881.175666809082,
      "loss": 2.965963840484619,
      "hidden": 0.17215122655034065,
      "layer": 0.08681348152458668,
      "mlm": 1.604857325553894,
      "distill": 0.9733972698450089,
      "teacher_acc": 0.6984842717647552
    },
    {
      "step": 13311,
      "lr": 3.579465685153416e-06,
      "elapsed_s": 9888.578418254852,
      "loss": 2.772668719291687,
      "hidden": 0.17213476449251175,
      "layer": 0.08514178358018398,
      "mlm": 1.4024863839149475,
      "distill": 0.9833419173955917,
      "teacher_acc": 0.7443005293607712
    },
    {
      "step": 13321,
      "lr": 3.5535500500193357e-06,
      "elapsed_s": 9895.972501516342,
      "loss": 2.8314099311828613,
      "hidden": 0.17797372862696648,
      "layer": 0.0872762780636549,
      "mlm": 1.438452959060669,
      "distill": 0.9933713376522064,
      "teacher_acc": 0.7434177845716476
    },
    {
      "step": 13331,
      "lr": 3.527721392163891e-06,
      "elapsed_s": 9903.37880063057,
      "loss": 2.8499123454093933,
      "hidden": 0.1729104109108448,
      "layer": 0.0865900106728077,
      "mlm": 1.5001056790351868,
      "distill": 0.9606907963752747,
      "teacher_acc": 0.714264452457428
    },
    {
      "step": 13341,
      "lr": 3.5019798163366424e-06,
      "elapsed_s": 9910.788428068161,
      "loss": 2.8279574513435364,
      "hidden": 0.18093464523553848,
      "layer": 0.0887424685060978,
      "mlm": 1.4483312666416168,
      "distill": 0.9733856469392776,
      "teacher_acc": 0.7274457663297653
    },
    {
      "step": 13351,
      "lr": 3.4763254269339963e-06,
      "elapsed_s": 9918.218132019043,
      "loss": 2.9681381583213806,
      "hidden": 0.18314063549041748,
      "layer": 0.08974025771021843,
      "mlm": 1.5136862993240356,
      "distill": 1.043300375342369,
      "teacher_acc": 0.7237362116575241
    },
    {
      "step": 13361,
      "lr": 3.450758327998768e-06,
      "elapsed_s": 9925.625757932663,
      "loss": 2.938211977481842,
      "hidden": 0.17523200437426567,
      "layer": 0.0886930376291275,
      "mlm": 1.4801146984100342,
      "distill": 1.063286766409874,
      "teacher_acc": 0.7262709587812424
    },
    {
      "step": 13371,
      "lr": 3.425278623219752e-06,
      "elapsed_s": 9933.037760972977,
      "loss": 2.9477908611297607,
      "hidden": 0.18083260208368301,
      "layer": 0.08948999457061291,
      "mlm": 1.5002566576004028,
      "distill": 1.0411240607500076,
      "teacher_acc": 0.7286197543144226
    },
    {
      "step": 13381,
      "lr": 3.3998864159313303e-06,
      "elapsed_s": 9940.452121019363,
      "loss": 2.8202080130577087,
      "hidden": 0.1717272363603115,
      "layer": 0.08626286685466766,
      "mlm": 1.5061494708061218,
      "distill": 0.9274726510047913,
      "teacher_acc": 0.7199335843324661
    },
    {
      "step": 13391,
      "lr": 3.3745818091130033e-06,
      "elapsed_s": 9947.876585245132,
      "loss": 2.9197599291801453,
      "hidden": 0.17536289989948273,
      "layer": 0.0863843560218811,
      "mlm": 1.553793340921402,
      "distill": 0.9720486104488373,
      "teacher_acc": 0.7109072804450989
    },
    {
      "step": 13401,
      "lr": 3.3493649053890326e-06,
      "elapsed_s": 9955.239881515503,
      "loss": 2.9269282817840576,
      "hidden": 0.17357299104332924,
      "layer": 0.08617493696510792,
      "mlm": 1.589437484741211,
      "distill": 0.947257325053215,
      "teacher_acc": 0.6985394358634949
    },
    {
      "step": 13411,
      "lr": 3.324235807027981e-06,
      "elapsed_s": 9962.644291639328,
      "loss": 2.8427634835243225,
      "hidden": 0.17464591562747955,
      "layer": 0.08647837489843369,
      "mlm": 1.4998052418231964,
      "distill": 0.950427234172821,
      "teacher_acc": 0.70854452252388
    },
    {
      "step": 13421,
      "lr": 3.2991946159423137e-06,
      "elapsed_s": 9970.025236606598,
      "loss": 2.842606246471405,
      "hidden": 0.173444963991642,
      "layer": 0.08649384044110775,
      "mlm": 1.5227293372154236,
      "distill": 0.9297401309013367,
      "teacher_acc": 0.7136987000703812
    },
    {
      "step": 13431,
      "lr": 3.2742414336879858e-06,
      "elapsed_s": 9977.41947054863,
      "loss": 2.6814911365509033,
      "hidden": 0.17017915472388268,
      "layer": 0.08438727632164955,
      "mlm": 1.3863603472709656,
      "distill": 0.912578895688057,
      "teacher_acc": 0.746785506606102
    },
    {
      "step": 13441,
      "lr": 3.249376361464021e-06,
      "elapsed_s": 9984.788188457489,
      "loss": 2.9087327122688293,
      "hidden": 0.17752645909786224,
      "layer": 0.08745256811380386,
      "mlm": 1.5356897413730621,
      "distill": 0.9742637425661087,
      "teacher_acc": 0.7085938304662704
    },
    {
      "step": 13451,
      "lr": 3.2245995001121106e-06,
      "elapsed_s": 9992.184267282486,
      "loss": 2.827235460281372,
      "hidden": 0.17676269263029099,
      "layer": 0.08634806796908379,
      "mlm": 1.4623760282993317,
      "distill": 0.968160018324852,
      "teacher_acc": 0.7254102379083633
    },
    {
      "step": 13461,
      "lr": 3.1999109501162084e-06,
      "elapsed_s": 9999.588663816452,
      "loss": 2.85288405418396,
      "hidden": 0.17578741908073425,
      "layer": 0.08693391643464565,
      "mlm": 1.4931911826133728,
      "distill": 0.9646510034799576,
      "teacher_acc": 0.7216839343309402
    },
    {
      "step": 13471,
      "lr": 3.1753108116021056e-06,
      "elapsed_s": 10007.00560426712,
      "loss": 3.0027015805244446,
      "hidden": 0.1762189380824566,
      "layer": 0.08733008429408073,
      "mlm": 1.5448495745658875,
      "distill": 1.0617490857839584,
      "teacher_acc": 0.722916767001152
    },
    {
      "step": 13481,
      "lr": 3.1507991843370526e-06,
      "elapsed_s": 10014.383801221848,
      "loss": 2.82051944732666,
      "hidden": 0.17620981857180595,
      "layer": 0.08739597909152508,
      "mlm": 1.4551913142204285,
      "distill": 0.9692105352878571,
      "teacher_acc": 0.7299276292324066
    },
    {
      "step": 13491,
      "lr": 3.126376167729314e-06,
      "elapsed_s": 10021.780824899673,
      "loss": 2.9078973531723022,
      "hidden": 0.17320147156715393,
      "layer": 0.08524545840919018,
      "mlm": 1.547870010137558,
      "distill": 0.9710016995668411,
      "teacher_acc": 0.7098441869020462
    },
    {
      "step": 13501,
      "lr": 3.1020418608278035e-06,
      "elapsed_s": 10029.217134714127,
      "loss": 2.7302801609039307,
      "hidden": 0.17064569145441055,
      "layer": 0.08427605405449867,
      "mlm": 1.4497428834438324,
      "distill": 0.8971079140901566,
      "teacher_acc": 0.7218370586633682
    },
    {
      "step": 13511,
      "lr": 3.0777963623216694e-06,
      "elapsed_s": 10036.617987394333,
      "loss": 2.8990476727485657,
      "hidden": 0.17579039558768272,
      "layer": 0.08751954883337021,
      "mlm": 1.506092131137848,
      "distill": 0.9976149499416351,
      "teacher_acc": 0.7273407280445099
    },
    {
      "step": 13521,
      "lr": 3.0536397705398843e-06,
      "elapsed_s": 10044.013816833496,
      "loss": 2.8781688809394836,
      "hidden": 0.1711338460445404,
      "layer": 0.08606979809701443,
      "mlm": 1.5276587009429932,
      "distill": 0.9652075469493866,
      "teacher_acc": 0.708276703953743
    },
    {
      "step": 13531,
      "lr": 3.029572183450868e-06,
      "elapsed_s": 10051.43296790123,
      "loss": 2.857636868953705,
      "hidden": 0.17460649833083153,
      "layer": 0.08687365986406803,
      "mlm": 1.5067120790481567,
      "distill": 0.9582749605178833,
      "teacher_acc": 0.711731493473053
    },
    {
      "step": 13541,
      "lr": 3.005593698662057e-06,
      "elapsed_s": 10058.829039812088,
      "loss": 2.8990264534950256,
      "hidden": 0.17268938571214676,
      "layer": 0.08606573939323425,
      "mlm": 1.5367178916931152,
      "distill": 0.9738968908786774,
      "teacher_acc": 0.7133716195821762
    },
    {
      "step": 13551,
      "lr": 2.9817044134195392e-06,
      "elapsed_s": 10066.232584953308,
      "loss": 2.840327560901642,
      "hidden": 0.170196533203125,
      "layer": 0.08610442839562893,
      "mlm": 1.5407018065452576,
      "distill": 0.9161804616451263,
      "teacher_acc": 0.7018955200910568
    },
    {
      "step": 13561,
      "lr": 2.957904424607652e-06,
      "elapsed_s": 10073.630909919739,
      "loss": 2.946484625339508,
      "hidden": 0.17957350984215736,
      "layer": 0.08602401986718178,
      "mlm": 1.5581636130809784,
      "distill": 0.986161932349205,
      "teacher_acc": 0.7082791328430176
    },
    {
      "step": 13571,
      "lr": 2.9341938287485855e-06,
      "elapsed_s": 10081.006314754486,
      "loss": 2.7946261763572693,
      "hidden": 0.17502402141690254,
      "layer": 0.08597644232213497,
      "mlm": 1.4548249542713165,
      "distill": 0.9467648863792419,
      "teacher_acc": 0.7259810417890549
    },
    {
      "step": 13581,
      "lr": 2.910572722001989e-06,
      "elapsed_s": 10088.381644487381,
      "loss": 2.718013107776642,
      "hidden": 0.17166883498430252,
      "layer": 0.08550783060491085,
      "mlm": 1.4195754826068878,
      "distill": 0.9123460799455643,
      "teacher_acc": 0.7344458103179932
    },
    {
      "step": 13591,
      "lr": 2.887041200164575e-06,
      "elapsed_s": 10095.75076842308,
      "loss": 2.830930471420288,
      "hidden": 0.17759772017598152,
      "layer": 0.08759403228759766,
      "mlm": 1.4853352010250092,
      "distill": 0.9466028809547424,
      "teacher_acc": 0.7255779206752777
    },
    {
      "step": 13601,
      "lr": 2.8635993586697553e-06,
      "elapsed_s": 10103.100806713104,
      "loss": 2.896745979785919,
      "hidden": 0.17577078938484192,
      "layer": 0.08677500858902931,
      "mlm": 1.486181527376175,
      "distill": 1.015635371208191,
      "teacher_acc": 0.7232370525598526
    },
    {
      "step": 13611,
      "lr": 2.8402472925872255e-06,
      "elapsed_s": 10110.491586208344,
      "loss": 2.8552364110946655,
      "hidden": 0.18045958876609802,
      "layer": 0.08799123950302601,
      "mlm": 1.4385304152965546,
      "distill": 1.011791169643402,
      "teacher_acc": 0.7440200001001358
    },
    {
      "step": 13621,
      "lr": 2.816985096622604e-06,
      "elapsed_s": 10117.87307214737,
      "loss": 2.953605592250824,
      "hidden": 0.1766708865761757,
      "layer": 0.08778412640094757,
      "mlm": 1.5450712442398071,
      "distill": 1.011300504207611,
      "teacher_acc": 0.7151249796152115
    },
    {
      "step": 13631,
      "lr": 2.7938128651170293e-06,
      "elapsed_s": 10125.293509244919,
      "loss": 2.8452564477920532,
      "hidden": 0.17177104577422142,
      "layer": 0.08644742332398891,
      "mlm": 1.5120657682418823,
      "distill": 0.9464248716831207,
      "teacher_acc": 0.7121517807245255
    },
    {
      "step": 13641,
      "lr": 2.770730692046775e-06,
      "elapsed_s": 10132.711966276169,
      "loss": 2.82638281583786,
      "hidden": 0.17850084602832794,
      "layer": 0.087546830996871,
      "mlm": 1.4528644979000092,
      "distill": 0.9727432876825333,
      "teacher_acc": 0.7360454350709915
    },
    {
      "step": 13651,
      "lr": 2.7477386710228895e-06,
      "elapsed_s": 10140.146907806396,
      "loss": 2.9267768263816833,
      "hidden": 0.18046989664435387,
      "layer": 0.08674029633402824,
      "mlm": 1.5328928530216217,
      "distill": 0.9895740002393723,
      "teacher_acc": 0.7141415476799011
    },
    {
      "step": 13661,
      "lr": 2.7248368952908053e-06,
      "elapsed_s": 10147.58401632309,
      "loss": 2.8222281336784363,
      "hidden": 0.1781761348247528,
      "layer": 0.08690999075770378,
      "mlm": 1.429492861032486,
      "distill": 0.9929278790950775,
      "teacher_acc": 0.7379884570837021
    },
    {
      "step": 13671,
      "lr": 2.7020254577299536e-06,
      "elapsed_s": 10154.977522850037,
      "loss": 2.8902623057365417,
      "hidden": 0.171976737678051,
      "layer": 0.08555602468550205,
      "mlm": 1.5623492002487183,
      "distill": 0.9411815851926804,
      "teacher_acc": 0.7009619623422623
    },
    {
      "step": 13681,
      "lr": 2.679304450853401e-06,
      "elapsed_s": 10162.384892463684,
      "loss": 2.808736562728882,
      "hidden": 0.1753615327179432,
      "layer": 0.08599212765693665,
      "mlm": 1.4499826431274414,
      "distill": 0.9650347828865051,
      "teacher_acc": 0.7319043576717377
    },
    {
      "step": 13691,
      "lr": 2.6566739668074576e-06,
      "elapsed_s": 10169.793513774872,
      "loss": 2.7754350900650024,
      "hidden": 0.1723586656153202,
      "layer": 0.08558506332337856,
      "mlm": 1.483440101146698,
      "distill": 0.9044850617647171,
      "teacher_acc": 0.7183823883533478
    },
    {
      "step": 13701,
      "lr": 2.6341340973713187e-06,
      "elapsed_s": 10177.182252883911,
      "loss": 2.778847575187683,
      "hidden": 0.17173828184604645,
      "layer": 0.08610547333955765,
      "mlm": 1.4377617239952087,
      "distill": 0.9545565545558929,
      "teacher_acc": 0.7317746132612228
    },
    {
      "step": 13711,
      "lr": 2.6116849339566875e-06,
      "elapsed_s": 10184.57947063446,
      "loss": 2.8364253640174866,
      "hidden": 0.1725776456296444,
      "layer": 0.08634312450885773,
      "mlm": 1.4619020521640778,
      "distill": 0.9861964881420135,
      "teacher_acc": 0.7254862040281296
    },
    {
      "step": 13721,
      "lr": 2.589326567607406e-06,
      "elapsed_s": 10191.950474262238,
      "loss": 2.7718119025230408,
      "hidden": 0.17714256793260574,
      "layer": 0.0865456685423851,
      "mlm": 1.4099999368190765,
      "distill": 0.9642539918422699,
      "teacher_acc": 0.7420271337032318
    },
    {
      "step": 13731,
      "lr": 2.5670590889990853e-06,
      "elapsed_s": 10199.346661806107,
      "loss": 2.8479841351509094,
      "hidden": 0.17484470829367638,
      "layer": 0.08732474967837334,
      "mlm": 1.4903290569782257,
      "distill": 0.9643032252788544,
      "teacher_acc": 0.722683846950531
    },
    {
      "step": 13741,
      "lr": 2.5448825884387185e-06,
      "elapsed_s": 10206.77306675911,
      "loss": 2.7772529125213623,
      "hidden": 0.17240336909890175,
      "layer": 0.08511606231331825,
      "mlm": 1.480447143316269,
      "distill": 0.9094410240650177,
      "teacher_acc": 0.706803634762764
    },
    {
      "step": 13751,
      "lr": 2.5227971558643537e-06,
      "elapsed_s": 10214.18965291977,
      "loss": 2.805159270763397,
      "hidden": 0.17355717718601227,
      "layer": 0.08706989884376526,
      "mlm": 1.46714249253273,
      "distill": 0.94736747443676,
      "teacher_acc": 0.729495644569397
    },
    {
      "step": 13761,
      "lr": 2.500802880844699e-06,
      "elapsed_s": 10221.603974342346,
      "loss": 2.8740991353988647,
      "hidden": 0.1765865683555603,
      "layer": 0.08629830181598663,
      "mlm": 1.4889123737812042,
      "distill": 0.9888645112514496,
      "teacher_acc": 0.7216536700725555
    },
    {
      "step": 13771,
      "lr": 2.478899852578767e-06,
      "elapsed_s": 10229.00546503067,
      "loss": 2.688551962375641,
      "hidden": 0.169419065117836,
      "layer": 0.08447246998548508,
      "mlm": 1.430046170949936,
      "distill": 0.8774314373731613,
      "teacher_acc": 0.7370067834854126
    },
    {
      "step": 13781,
      "lr": 2.457088159895518e-06,
      "elapsed_s": 10236.423662900925,
      "loss": 2.9424074292182922,
      "hidden": 0.17341970652341843,
      "layer": 0.08687831647694111,
      "mlm": 1.5430165231227875,
      "distill": 1.0091122835874557,
      "teacher_acc": 0.7156349718570709
    },
    {
      "step": 13791,
      "lr": 2.43536789125349e-06,
      "elapsed_s": 10243.854454755783,
      "loss": 2.9735423922538757,
      "hidden": 0.1810770407319069,
      "layer": 0.08804386854171753,
      "mlm": 1.537389189004898,
      "distill": 1.0299771279096603,
      "teacher_acc": 0.7164116054773331
    },
    {
      "step": 13801,
      "lr": 2.4137391347404476e-06,
      "elapsed_s": 10251.289105176926,
      "loss": 2.8571659326553345,
      "hidden": 0.17165841534733772,
      "layer": 0.08736958168447018,
      "mlm": 1.468131422996521,
      "distill": 1.002032846212387,
      "teacher_acc": 0.73350790143013
    },
    {
      "step": 13811,
      "lr": 2.3922019780730192e-06,
      "elapsed_s": 10258.710137844086,
      "loss": 2.8679582476615906,
      "hidden": 0.17781966924667358,
      "layer": 0.08721243776381016,
      "mlm": 1.468209058046341,
      "distill": 1.0005036294460297,
      "teacher_acc": 0.7386182695627213
    },
    {
      "step": 13821,
      "lr": 2.3707565085963667e-06,
      "elapsed_s": 10266.125086784363,
      "loss": 2.973024010658264,
      "hidden": 0.17217721417546272,
      "layer": 0.08818310126662254,
      "mlm": 1.5950253307819366,
      "distill": 0.9895526319742203,
      "teacher_acc": 0.7040073573589325
    },
    {
      "step": 13831,
      "lr": 2.3494028132837796e-06,
      "elapsed_s": 10273.559455394745,
      "loss": 2.7924931049346924,
      "hidden": 0.17540602385997772,
      "layer": 0.08654344268143177,
      "mlm": 1.4439668357372284,
      "distill": 0.954442486166954,
      "teacher_acc": 0.7309968918561935
    },
    {
      "step": 13841,
      "lr": 2.328140978736365e-06,
      "elapsed_s": 10280.958008050919,
      "loss": 2.9041112661361694,
      "hidden": 0.1775062046945095,
      "layer": 0.08754895254969597,
      "mlm": 1.4747491478919983,
      "distill": 1.0305751711130142,
      "teacher_acc": 0.7350284904241562
    },
    {
      "step": 13851,
      "lr": 2.306971091182686e-06,
      "elapsed_s": 10288.371163845062,
      "loss": 2.7793145179748535,
      "hidden": 0.1688280776143074,
      "layer": 0.08618584834039211,
      "mlm": 1.5187110006809235,
      "distill": 0.8798544555902481,
      "teacher_acc": 0.7138697952032089
    },
    {
      "step": 13861,
      "lr": 2.2858932364784034e-06,
      "elapsed_s": 10295.801223754883,
      "loss": 2.8965718746185303,
      "hidden": 0.1723787970840931,
      "layer": 0.08558940514922142,
      "mlm": 1.5538792312145233,
      "distill": 0.9551402628421783,
      "teacher_acc": 0.7150238901376724
    },
    {
      "step": 13871,
      "lr": 2.264907500105945e-06,
      "elapsed_s": 10303.24336218834,
      "loss": 2.8284568190574646,
      "hidden": 0.18051692470908165,
      "layer": 0.08728775195777416,
      "mlm": 1.4243427217006683,
      "distill": 0.9994364231824875,
      "teacher_acc": 0.7403854727745056
    },
    {
      "step": 13881,
      "lr": 2.2440139671741356e-06,
      "elapsed_s": 10310.655108690262,
      "loss": 2.83755362033844,
      "hidden": 0.17258794978260994,
      "layer": 0.08561290428042412,
      "mlm": 1.5178918838500977,
      "distill": 0.9316794127225876,
      "teacher_acc": 0.7169499397277832
    },
    {
      "step": 13891,
      "lr": 2.2232127224178637e-06,
      "elapsed_s": 10318.077113389969,
      "loss": 2.7351359724998474,
      "hidden": 0.17573755607008934,
      "layer": 0.08525663055479527,
      "mlm": 1.417270541191101,
      "distill": 0.9237620085477829,
      "teacher_acc": 0.7255730330944061
    },
    {
      "step": 13901,
      "lr": 2.2025038501977486e-06,
      "elapsed_s": 10325.500655412674,
      "loss": 2.7581538558006287,
      "hidden": 0.170022152364254,
      "layer": 0.084604537114501,
      "mlm": 1.456235557794571,
      "distill": 0.9195716977119446,
      "teacher_acc": 0.732001006603241
    },
    {
      "step": 13911,
      "lr": 2.181887434499774e-06,
      "elapsed_s": 10332.914532661438,
      "loss": 2.786330759525299,
      "hidden": 0.172620490193367,
      "layer": 0.0852566733956337,
      "mlm": 1.4611341655254364,
      "distill": 0.9373273104429245,
      "teacher_acc": 0.7210332453250885
    },
    {
      "step": 13921,
      "lr": 2.1613635589349756e-06,
      "elapsed_s": 10340.322680950165,
      "loss": 2.7879751920700073,
      "hidden": 0.1731111891567707,
      "layer": 0.08607964403927326,
      "mlm": 1.5158867537975311,
      "distill": 0.8828262537717819,
      "teacher_acc": 0.7099797278642654
    },
    {
      "step": 13931,
      "lr": 2.1409323067390797e-06,
      "elapsed_s": 10347.730446577072,
      "loss": 2.837278425693512,
      "hidden": 0.1709444634616375,
      "layer": 0.08515521138906479,
      "mlm": 1.5361400544643402,
      "distill": 0.9166718423366547,
      "teacher_acc": 0.7014962583780289
    },
    {
      "step": 13941,
      "lr": 2.120593760772166e-06,
      "elapsed_s": 10355.104372262955,
      "loss": 2.796305000782013,
      "hidden": 0.16753742098808289,
      "layer": 0.08469321765005589,
      "mlm": 1.5235105156898499,
      "distill": 0.8953730314970016,
      "teacher_acc": 0.7005200535058975
    },
    {
      "step": 13951,
      "lr": 2.1003480035183513e-06,
      "elapsed_s": 10362.49483013153,
      "loss": 2.8059089183807373,
      "hidden": 0.16784042865037918,
      "layer": 0.08432520367205143,
      "mlm": 1.5245665311813354,
      "distill": 0.9034988880157471,
      "teacher_acc": 0.7140496075153351
    },
    {
      "step": 13961,
      "lr": 2.08019511708544e-06,
      "elapsed_s": 10369.905203342438,
      "loss": 2.885190486907959,
      "hidden": 0.17706922814249992,
      "layer": 0.08653123304247856,
      "mlm": 1.5224555134773254,
      "distill": 0.9653308540582657,
      "teacher_acc": 0.7140603810548782
    },
    {
      "step": 13971,
      "lr": 2.0601351832045866e-06,
      "elapsed_s": 10377.315805196762,
      "loss": 2.7942179441452026,
      "hidden": 0.1794845461845398,
      "layer": 0.0889893937855959,
      "mlm": 1.4326823055744171,
      "distill": 0.9580718129873276,
      "teacher_acc": 0.7354013472795486
    },
    {
      "step": 13981,
      "lr": 2.0401682832299845e-06,
      "elapsed_s": 10384.733602285385,
      "loss": 2.793272912502289,
      "hidden": 0.17557357624173164,
      "layer": 0.08620630577206612,
      "mlm": 1.447041541337967,
      "distill": 0.9519810527563095,
      "teacher_acc": 0.7309431284666061
    },
    {
      "step": 13991,
      "lr": 2.020294498138503e-06,
      "elapsed_s": 10392.12751698494,
      "loss": 2.8158835768699646,
      "hidden": 0.17689795047044754,
      "layer": 0.08673472329974174,
      "mlm": 1.4679576456546783,
      "distill": 0.9507626444101334,
      "teacher_acc": 0.7258395254611969
    },
    {
      "step": 14001,
      "lr": 2.0005139085293945e-06,
      "elapsed_s": 10401.608981370926,
      "loss": 2.76169216632843,
      "hidden": 0.16610056906938553,
      "layer": 0.08407432027161121,
      "mlm": 1.46820867061615,
      "distill": 0.9192452281713486,
      "teacher_acc": 0.7223249077796936
    },
    {
      "step": 14011,
      "lr": 1.9808265946239523e-06,
      "elapsed_s": 10409.022718429565,
      "loss": 2.802949070930481,
      "hidden": 0.1710244081914425,
      "layer": 0.08679200895130634,
      "mlm": 1.4966979920864105,
      "distill": 0.9208062589168549,
      "teacher_acc": 0.7197669893503189
    },
    {
      "step": 14021,
      "lr": 1.9612326362651757e-06,
      "elapsed_s": 10416.40318441391,
      "loss": 2.794956624507904,
      "hidden": 0.1750604845583439,
      "layer": 0.08662434853613377,
      "mlm": 1.4720516502857208,
      "distill": 0.9294718205928802,
      "teacher_acc": 0.7237173467874527
    },
    {
      "step": 14031,
      "lr": 1.941732112917466e-06,
      "elapsed_s": 10423.815347194672,
      "loss": 2.929684281349182,
      "hidden": 0.17674985900521278,
      "layer": 0.08677071332931519,
      "mlm": 1.5664366483688354,
      "distill": 0.9663625955581665,
      "teacher_acc": 0.705165222287178
    },
    {
      "step": 14041,
      "lr": 1.922325103666281e-06,
      "elapsed_s": 10431.199553966522,
      "loss": 2.904448091983795,
      "hidden": 0.1739608570933342,
      "layer": 0.08673299476504326,
      "mlm": 1.5817512273788452,
      "distill": 0.9314086586236954,
      "teacher_acc": 0.7042127847671509
    },
    {
      "step": 14051,
      "lr": 1.9030116872178316e-06,
      "elapsed_s": 10438.59989619255,
      "loss": 2.752030670642853,
      "hidden": 0.17183400318026543,
      "layer": 0.08621217496693134,
      "mlm": 1.449620246887207,
      "distill": 0.9156363606452942,
      "teacher_acc": 0.7328990399837494
    },
    {
      "step": 14061,
      "lr": 1.8837919418987659e-06,
      "elapsed_s": 10445.975409269333,
      "loss": 2.7731663584709167,
      "hidden": 0.17027044668793678,
      "layer": 0.084932342171669,
      "mlm": 1.493860274553299,
      "distill": 0.896298959851265,
      "teacher_acc": 0.7175787091255188
    },
    {
      "step": 14071,
      "lr": 1.8646659456558357e-06,
      "elapsed_s": 10453.361553668976,
      "loss": 2.873856484889984,
      "hidden": 0.17382044717669487,
      "layer": 0.08752700127661228,
      "mlm": 1.5031442642211914,
      "distill": 0.9793078303337097,
      "teacher_acc": 0.7198782861232758
    },
    {
      "step": 14081,
      "lr": 1.8456337760555915e-06,
      "elapsed_s": 10460.736646175385,
      "loss": 2.745754837989807,
      "hidden": 0.1751195378601551,
      "layer": 0.08550835400819778,
      "mlm": 1.4350703358650208,
      "distill": 0.9176911860704422,
      "teacher_acc": 0.731780931353569
    },
    {
      "step": 14091,
      "lr": 1.8266955102840544e-06,
      "elapsed_s": 10468.110454559326,
      "loss": 3.016284704208374,
      "hidden": 0.17835402861237526,
      "layer": 0.08700285479426384,
      "mlm": 1.5910156667232513,
      "distill": 1.0250595957040787,
      "teacher_acc": 0.7032258063554764
    },
    {
      "step": 14101,
      "lr": 1.8078512251464286e-06,
      "elapsed_s": 10475.516586065292,
      "loss": 2.8399804830551147,
      "hidden": 0.16947676241397858,
      "layer": 0.08494937233626842,
      "mlm": 1.5065691471099854,
      "distill": 0.9519830495119095,
      "teacher_acc": 0.7126374989748001
    },
    {
      "step": 14111,
      "lr": 1.7891009970667672e-06,
      "elapsed_s": 10482.91712474823,
      "loss": 2.9532009959220886,
      "hidden": 0.17750415205955505,
      "layer": 0.08710438385605812,
      "mlm": 1.5591579973697662,
      "distill": 0.9954824596643448,
      "teacher_acc": 0.7058866769075394
    },
    {
      "step": 14121,
      "lr": 1.77044490208767e-06,
      "elapsed_s": 10490.346314907074,
      "loss": 2.7569345831871033,
      "hidden": 0.1716279461979866,
      "layer": 0.0856749676167965,
      "mlm": 1.4318766295909882,
      "distill": 0.9389645159244537,
      "teacher_acc": 0.7311614155769348
    },
    {
      "step": 14131,
      "lr": 1.7518830158699761e-06,
      "elapsed_s": 10497.791120767593,
      "loss": 2.7607027292251587,
      "hidden": 0.17083078250288963,
      "layer": 0.08581781387329102,
      "mlm": 1.4493281245231628,
      "distill": 0.9268041253089905,
      "teacher_acc": 0.7253232151269913
    },
    {
      "step": 14141,
      "lr": 1.7334154136924491e-06,
      "elapsed_s": 10505.176270246506,
      "loss": 2.799915075302124,
      "hidden": 0.174549151211977,
      "layer": 0.08530793711543083,
      "mlm": 1.4337543547153473,
      "distill": 0.9744085073471069,
      "teacher_acc": 0.7375188320875168
    },
    {
      "step": 14151,
      "lr": 1.7150421704514863e-06,
      "elapsed_s": 10512.559198856354,
      "loss": 2.733894169330597,
      "hidden": 0.17086751759052277,
      "layer": 0.08504052460193634,
      "mlm": 1.4350909292697906,
      "distill": 0.9145479649305344,
      "teacher_acc": 0.7287153452634811
    },
    {
      "step": 14161,
      "lr": 1.696763360660808e-06,
      "elapsed_s": 10519.944802999496,
      "loss": 2.8709142208099365,
      "hidden": 0.17224057763814926,
      "layer": 0.08669607527554035,
      "mlm": 1.5268691182136536,
      "distill": 0.9562158435583115,
      "teacher_acc": 0.7134450227022171
    },
    {
      "step": 14171,
      "lr": 1.6785790584511435e-06,
      "elapsed_s": 10527.368250131607,
      "loss": 2.822826087474823,
      "hidden": 0.17584363371133804,
      "layer": 0.08717238530516624,
      "mlm": 1.4723452627658844,
      "distill": 0.9552074074745178,
      "teacher_acc": 0.7209930568933487
    },
    {
      "step": 14181,
      "lr": 1.6604893375699594e-06,
      "elapsed_s": 10534.792336940765,
      "loss": 2.793032944202423,
      "hidden": 0.17144005373120308,
      "layer": 0.08522113040089607,
      "mlm": 1.4916033446788788,
      "distill": 0.9159389138221741,
      "teacher_acc": 0.7202620506286621
    },
    {
      "step": 14191,
      "lr": 1.6424942713811204e-06,
      "elapsed_s": 10542.209605932236,
      "loss": 2.7623318433761597,
      "hidden": 0.17746607214212418,
      "layer": 0.0878321249037981,
      "mlm": 1.4097042679786682,
      "distill": 0.9537792950868607,
      "teacher_acc": 0.7382763624191284
    },
    {
      "step": 14201,
      "lr": 1.624593932864632e-06,
      "elapsed_s": 10549.643550634384,
      "loss": 2.880506217479706,
      "hidden": 0.17743392288684845,
      "layer": 0.0868946835398674,
      "mlm": 1.483972042798996,
      "distill": 0.9982190579175949,
      "teacher_acc": 0.7198678553104401
    },
    {
      "step": 14211,
      "lr": 1.6067883946163154e-06,
      "elapsed_s": 10557.080504179,
      "loss": 2.8627567887306213,
      "hidden": 0.1716657169163227,
      "layer": 0.08384089544415474,
      "mlm": 1.54677215218544,
      "distill": 0.9307327121496201,
      "teacher_acc": 0.7150154411792755
    },
    {
      "step": 14221,
      "lr": 1.5890777288475296e-06,
      "elapsed_s": 10564.52091050148,
      "loss": 2.8106547594070435,
      "hidden": 0.16821079328656197,
      "layer": 0.08497673831880093,
      "mlm": 1.465882569551468,
      "distill": 0.9658621996641159,
      "teacher_acc": 0.7320643067359924
    },
    {
      "step": 14231,
      "lr": 1.5714620073848774e-06,
      "elapsed_s": 10571.960143327713,
      "loss": 2.806226134300232,
      "hidden": 0.1807350590825081,
      "layer": 0.08767815679311752,
      "mlm": 1.404232770204544,
      "distill": 0.996684119105339,
      "teacher_acc": 0.7395389527082443
    },
    {
      "step": 14241,
      "lr": 1.5539413016698922e-06,
      "elapsed_s": 10579.395082712173,
      "loss": 2.6752718687057495,
      "hidden": 0.1680995263159275,
      "layer": 0.08456046134233475,
      "mlm": 1.4084307849407196,
      "distill": 0.8883618414402008,
      "teacher_acc": 0.7343160659074783
    },
    {
      "step": 14251,
      "lr": 1.5365156827587822e-06,
      "elapsed_s": 10586.809281349182,
      "loss": 2.7044894099235535,
      "hidden": 0.17138204351067543,
      "layer": 0.08655286207795143,
      "mlm": 1.4075784385204315,
      "distill": 0.9108704477548599,
      "teacher_acc": 0.7320179343223572
    },
    {
      "step": 14261,
      "lr": 1.5191852213221197e-06,
      "elapsed_s": 10594.208156824112,
      "loss": 2.729857385158539,
      "hidden": 0.17048194631934166,
      "layer": 0.08541163802146912,
      "mlm": 1.4426599442958832,
      "distill": 0.903527706861496,
      "teacher_acc": 0.7260657995939255
    },
    {
      "step": 14271,
      "lr": 1.501949987644566e-06,
      "elapsed_s": 10601.633776426315,
      "loss": 2.897200345993042,
      "hidden": 0.17486248165369034,
      "layer": 0.08685139194130898,
      "mlm": 1.4816516041755676,
      "distill": 1.022398054599762,
      "teacher_acc": 0.7214303314685822
    },
    {
      "step": 14281,
      "lr": 1.4848100516245717e-06,
      "elapsed_s": 10609.034410476685,
      "loss": 2.7583463191986084,
      "hidden": 0.17370961606502533,
      "layer": 0.08594811521470547,
      "mlm": 1.408496618270874,
      "distill": 0.9594565182924271,
      "teacher_acc": 0.7314571589231491
    },
    {
      "step": 14291,
      "lr": 1.4677654827741111e-06,
      "elapsed_s": 10616.44923901558,
      "loss": 2.8035049438476562,
      "hidden": 0.17421622574329376,
      "layer": 0.08625682070851326,
      "mlm": 1.4719065427780151,
      "distill": 0.9400375336408615,
      "teacher_acc": 0.7200664430856705
    },
    {
      "step": 14301,
      "lr": 1.4508163502183786e-06,
      "elapsed_s": 10623.89735174179,
      "loss": 2.8051029443740845,
      "hidden": 0.1713331751525402,
      "layer": 0.08489186689257622,
      "mlm": 1.5115172863006592,
      "distill": 0.9084734320640564,
      "teacher_acc": 0.7066345363855362
    },
    {
      "step": 14311,
      "lr": 1.4339627226955392e-06,
      "elapsed_s": 10631.30501484871,
      "loss": 2.760395646095276,
      "hidden": 0.17375002056360245,
      "layer": 0.08588007092475891,
      "mlm": 1.4438023269176483,
      "distill": 0.9261532723903656,
      "teacher_acc": 0.7217451930046082
    },
    {
      "step": 14321,
      "lr": 1.4172046685564212e-06,
      "elapsed_s": 10638.763637304306,
      "loss": 2.872941493988037,
      "hidden": 0.17066996544599533,
      "layer": 0.08552526123821735,
      "mlm": 1.515047937631607,
      "distill": 0.9737909883260727,
      "teacher_acc": 0.7192364633083344
    },
    {
      "step": 14331,
      "lr": 1.4005422557642533e-06,
      "elapsed_s": 10646.162722587585,
      "loss": 2.828407406806946,
      "hidden": 0.17114480957388878,
      "layer": 0.08599905669689178,
      "mlm": 1.498354583978653,
      "distill": 0.9447636604309082,
      "teacher_acc": 0.7211867421865463
    },
    {
      "step": 14341,
      "lr": 1.3839755518943786e-06,
      "elapsed_s": 10653.594453811646,
      "loss": 2.8147324323654175,
      "hidden": 0.17176616936922073,
      "layer": 0.08574691228568554,
      "mlm": 1.5120875537395477,
      "distill": 0.9162391424179077,
      "teacher_acc": 0.7049439400434494
    },
    {
      "step": 14351,
      "lr": 1.3675046241339918e-06,
      "elapsed_s": 10661.027324438095,
      "loss": 2.929498076438904,
      "hidden": 0.18100519850850105,
      "layer": 0.08845037780702114,
      "mlm": 1.4862072467803955,
      "distill": 1.037055253982544,
      "teacher_acc": 0.7324432730674744
    },
    {
      "step": 14361,
      "lr": 1.35112953928187e-06,
      "elapsed_s": 10668.443279027939,
      "loss": 2.8735697865486145,
      "hidden": 0.1717100478708744,
      "layer": 0.08651827462017536,
      "mlm": 1.522817313671112,
      "distill": 0.9640732109546661,
      "teacher_acc": 0.713957279920578
    },
    {
      "step": 14371,
      "lr": 1.3348503637480774e-06,
      "elapsed_s": 10675.880018234253,
      "loss": 2.7407710552215576,
      "hidden": 0.1716291680932045,
      "layer": 0.08558456785976887,
      "mlm": 1.4371619820594788,
      "distill": 0.9175583869218826,
      "teacher_acc": 0.7340006083250046
    },
    {
      "step": 14381,
      "lr": 1.318667163553733e-06,
      "elapsed_s": 10683.281696796417,
      "loss": 2.8329426646232605,
      "hidden": 0.1754225268959999,
      "layer": 0.08518490754067898,
      "mlm": 1.4657265543937683,
      "distill": 0.973778635263443,
      "teacher_acc": 0.7269022464752197
    },
    {
      "step": 14391,
      "lr": 1.302580004330703e-06,
      "elapsed_s": 10690.667478322983,
      "loss": 2.9925026297569275,
      "hidden": 0.18222379684448242,
      "layer": 0.08886671997606754,
      "mlm": 1.5541399419307709,
      "distill": 1.0294816046953201,
      "teacher_acc": 0.7146477401256561
    },
    {
      "step": 14401,
      "lr": 1.286588951321363e-06,
      "elapsed_s": 10698.068416595459,
      "loss": 2.7938133478164673,
      "hidden": 0.17523540556430817,
      "layer": 0.0857910756021738,
      "mlm": 1.4221921563148499,
      "distill": 0.9782548099756241,
      "teacher_acc": 0.7374463826417923
    },
    {
      "step": 14411,
      "lr": 1.2706940693783225e-06,
      "elapsed_s": 10705.505653381348,
      "loss": 2.7076727747917175,
      "hidden": 0.16958023980259895,
      "layer": 0.08545582368969917,
      "mlm": 1.4056776762008667,
      "distill": 0.9201067388057709,
      "teacher_acc": 0.7409806102514267
    },
    {
      "step": 14421,
      "lr": 1.2548954229641725e-06,
      "elapsed_s": 10712.97174334526,
      "loss": 2.864917516708374,
      "hidden": 0.1760558933019638,
      "layer": 0.08566531166434288,
      "mlm": 1.5528311729431152,
      "distill": 0.9171419441699982,
      "teacher_acc": 0.7019470483064651
    },
    {
      "step": 14431,
      "lr": 1.2391930761512045e-06,
      "elapsed_s": 10720.386365175247,
      "loss": 2.7830395698547363,
      "hidden": 0.17527521774172783,
      "layer": 0.08628619275987148,
      "mlm": 1.4018632173538208,
      "distill": 0.9874828457832336,
      "teacher_acc": 0.7434203028678894
    },
    {
      "step": 14441,
      "lr": 1.2235870926211619e-06,
      "elapsed_s": 10727.794499397278,
      "loss": 2.8224573135375977,
      "hidden": 0.17201048880815506,
      "layer": 0.08530403487384319,
      "mlm": 1.4746313691139221,
      "distill": 0.961152970790863,
      "teacher_acc": 0.7275345772504807
    },
    {
      "step": 14451,
      "lr": 1.2080775356649866e-06,
      "elapsed_s": 10735.195686101913,
      "loss": 2.8824841380119324,
      "hidden": 0.1731123961508274,
      "layer": 0.08634190075099468,
      "mlm": 1.5141050517559052,
      "distill": 0.9789833575487137,
      "teacher_acc": 0.7098517864942551
    },
    {
      "step": 14461,
      "lr": 1.1926644681825556e-06,
      "elapsed_s": 10742.586343765259,
      "loss": 2.897954821586609,
      "hidden": 0.17373031377792358,
      "layer": 0.0867721289396286,
      "mlm": 1.5148891806602478,
      "distill": 0.9922189712524414,
      "teacher_acc": 0.7212705314159393
    },
    {
      "step": 14471,
      "lr": 1.1773479526824282e-06,
      "elapsed_s": 10750.023885250092,
      "loss": 2.8197315335273743,
      "hidden": 0.17528011277318,
      "layer": 0.08631559833884239,
      "mlm": 1.4834266901016235,
      "distill": 0.9425867944955826,
      "teacher_acc": 0.719948410987854
    },
    {
      "step": 14481,
      "lr": 1.162128051281594e-06,
      "elapsed_s": 10757.447720766068,
      "loss": 2.881897807121277,
      "hidden": 0.17308423295617104,
      "layer": 0.0865514948964119,
      "mlm": 1.5254969596862793,
      "distill": 0.9669566005468369,
      "teacher_acc": 0.7233994603157043
    },
    {
      "step": 14491,
      "lr": 1.1470048257052085e-06,
      "elapsed_s": 10764.848556756973,
      "loss": 2.8593088388442993,
      "hidden": 0.17850501090288162,
      "layer": 0.08657425083220005,
      "mlm": 1.4811260104179382,
      "distill": 0.9778856635093689,
      "teacher_acc": 0.7271009534597397
    },
    {
      "step": 14501,
      "lr": 1.1319783372863602e-06,
      "elapsed_s": 10772.25808763504,
      "loss": 2.786432921886444,
      "hidden": 0.17393088713288307,
      "layer": 0.08689089305698872,
      "mlm": 1.462539255619049,
      "distill": 0.9325863867998123,
      "teacher_acc": 0.7282358258962631
    },
    {
      "step": 14511,
      "lr": 1.1170486469658182e-06,
      "elapsed_s": 10779.63590502739,
      "loss": 2.821333885192871,
      "hidden": 0.16686134040355682,
      "layer": 0.08442057855427265,
      "mlm": 1.6068719029426575,
      "distill": 0.838528960943222,
      "teacher_acc": 0.6733581721782684
    },
    {
      "step": 14521,
      "lr": 1.102215815291774e-06,
      "elapsed_s": 10787.04955625534,
      "loss": 2.856306552886963,
      "hidden": 0.17236362025141716,
      "layer": 0.08624440431594849,
      "mlm": 1.4837678968906403,
      "distill": 0.9846892058849335,
      "teacher_acc": 0.7233289629220963
    },
    {
      "step": 14531,
      "lr": 1.0874799024196137e-06,
      "elapsed_s": 10794.463998317719,
      "loss": 2.823406457901001,
      "hidden": 0.17308861017227173,
      "layer": 0.08762641064822674,
      "mlm": 1.4940694570541382,
      "distill": 0.9393465220928192,
      "teacher_acc": 0.7214764058589935
    },
    {
      "step": 14541,
      "lr": 1.0728409681116492e-06,
      "elapsed_s": 10801.889160394669,
      "loss": 2.69081974029541,
      "hidden": 0.17080075666308403,
      "layer": 0.08561859652400017,
      "mlm": 1.4093118011951447,
      "distill": 0.8970970958471298,
      "teacher_acc": 0.7368239611387253
    },
    {
      "step": 14551,
      "lr": 1.0582990717369013e-06,
      "elapsed_s": 10809.300548791885,
      "loss": 2.764712929725647,
      "hidden": 0.17329735308885574,
      "layer": 0.08581766672432423,
      "mlm": 1.4008901715278625,
      "distill": 0.9743192344903946,
      "teacher_acc": 0.7484188824892044
    },
    {
      "step": 14561,
      "lr": 1.0438542722708445e-06,
      "elapsed_s": 10816.719276428223,
      "loss": 2.9092506766319275,
      "hidden": 0.1757342740893364,
      "layer": 0.0870184637606144,
      "mlm": 1.5078702867031097,
      "distill": 1.006402626633644,
      "teacher_acc": 0.7252167463302612
    },
    {
      "step": 14571,
      "lr": 1.0295066282951738e-06,
      "elapsed_s": 10824.136897087097,
      "loss": 2.815778911113739,
      "hidden": 0.17264306545257568,
      "layer": 0.08592127077281475,
      "mlm": 1.4873088002204895,
      "distill": 0.9402233511209488,
      "teacher_acc": 0.7176450937986374
    },
    {
      "step": 14581,
      "lr": 1.0152561979975633e-06,
      "elapsed_s": 10831.550125837326,
      "loss": 2.8162782788276672,
      "hidden": 0.17506588250398636,
      "layer": 0.08703228645026684,
      "mlm": 1.4762937128543854,
      "distill": 0.9463365972042084,
      "teacher_acc": 0.7226410508155823
    },
    {
      "step": 14591,
      "lr": 1.0011030391714278e-06,
      "elapsed_s": 10838.968429327011,
      "loss": 2.858545958995819,
      "hidden": 0.17958089336752892,
      "layer": 0.08726862072944641,
      "mlm": 1.490052878856659,
      "distill": 0.9656970202922821,
      "teacher_acc": 0.7152596712112427
    },
    {
      "step": 14601,
      "lr": 9.87047209215694e-07,
      "elapsed_s": 10846.409361362457,
      "loss": 2.786189079284668,
      "hidden": 0.1754492148756981,
      "layer": 0.08665314316749573,
      "mlm": 1.4200316369533539,
      "distill": 0.9719324111938477,
      "teacher_acc": 0.7383500933647156
    },
    {
      "step": 14611,
      "lr": 9.730887651345722e-07,
      "elapsed_s": 10853.813233852386,
      "loss": 2.861258029937744,
      "hidden": 0.1722641997039318,
      "layer": 0.08585125394165516,
      "mlm": 1.5347341895103455,
      "distill": 0.9390698820352554,
      "teacher_acc": 0.7050961554050446
    },
    {
      "step": 14621,
      "lr": 9.592277635373097e-07,
      "elapsed_s": 10861.250692129135,
      "loss": 2.7728896141052246,
      "hidden": 0.16919482126832008,
      "layer": 0.08548467978835106,
      "mlm": 1.5000942945480347,
      "distill": 0.8916633725166321,
      "teacher_acc": 0.7130042463541031
    },
    {
      "step": 14631,
      "lr": 9.454642606379793e-07,
      "elapsed_s": 10868.68359208107,
      "loss": 2.751961410045624,
      "hidden": 0.17055664956569672,
      "layer": 0.08580171316862106,
      "mlm": 1.4170801937580109,
      "distill": 0.9508669674396515,
      "teacher_acc": 0.7317867428064346
    },
    {
      "step": 14641,
      "lr": 9.317983122552332e-07,
      "elapsed_s": 10876.109104633331,
      "loss": 2.782758414745331,
      "hidden": 0.16910052672028542,
      "layer": 0.08481230586767197,
      "mlm": 1.4876843988895416,
      "distill": 0.9144667834043503,
      "teacher_acc": 0.7128712385892868
    },
    {
      "step": 14651,
      "lr": 9.182299738120931e-07,
      "elapsed_s": 10883.49386048317,
      "loss": 2.7592004537582397,
      "hidden": 0.17571571096777916,
      "layer": 0.08717487752437592,
      "mlm": 1.4334011673927307,
      "distill": 0.9307804256677628,
      "teacher_acc": 0.7200534790754318
    },
    {
      "step": 14661,
      "lr": 9.047593003357141e-07,
      "elapsed_s": 10890.887219667435,
      "loss": 2.869601309299469,
      "hidden": 0.1723918840289116,
      "layer": 0.08528394065797329,
      "mlm": 1.539952278137207,
      "distill": 0.9422232508659363,
      "teacher_acc": 0.717275008559227
    },
    {
      "step": 14671,
      "lr": 8.913863464571654e-07,
      "elapsed_s": 10898.248170375824,
      "loss": 2.8651984333992004,
      "hidden": 0.17677663639187813,
      "layer": 0.08796209469437599,
      "mlm": 1.5254859030246735,
      "distill": 0.9421782791614532,
      "teacher_acc": 0.7144770920276642
    },
    {
      "step": 14681,
      "lr": 8.781111664112162e-07,
      "elapsed_s": 10905.6598072052,
      "loss": 2.860907196998596,
      "hidden": 0.17617647722363472,
      "layer": 0.08662915043532848,
      "mlm": 1.488585501909256,
      "distill": 0.976654127240181,
      "teacher_acc": 0.7255280166864395
    },
    {
      "step": 14691,
      "lr": 8.649338140360952e-07,
      "elapsed_s": 10913.085814237595,
      "loss": 2.8184260725975037,
      "hidden": 0.17308060079813004,
      "layer": 0.08560461550951004,
      "mlm": 1.4963504076004028,
      "distill": 0.9331120997667313,
      "teacher_acc": 0.7166269421577454
    },
    {
      "step": 14701,
      "lr": 8.51854342773295e-07,
      "elapsed_s": 10920.47568154335,
      "loss": 2.766369044780731,
      "hidden": 0.17789726331830025,
      "layer": 0.08677062578499317,
      "mlm": 1.4088303744792938,
      "distill": 0.9583588093519211,
      "teacher_acc": 0.7401765882968903
    },
    {
      "step": 14711,
      "lr": 8.388728056673428e-07,
      "elapsed_s": 10927.87915802002,
      "loss": 2.785901188850403,
      "hidden": 0.17261191457509995,
      "layer": 0.08565591648221016,
      "mlm": 1.4966124594211578,
      "distill": 0.9012369513511658,
      "teacher_acc": 0.7152309417724609
    },
    {
      "step": 14721,
      "lr": 8.259892553655946e-07,
      "elapsed_s": 10935.32442164421,
      "loss": 2.742553412914276,
      "hidden": 0.1710566058754921,
      "layer": 0.08520496636629105,
      "mlm": 1.4401735067367554,
      "distill": 0.9176642894744873,
      "teacher_acc": 0.7316845804452896
    },
    {
      "step": 14731,
      "lr": 8.132037441180046e-07,
      "elapsed_s": 10942.72343301773,
      "loss": 2.9203909635543823,
      "hidden": 0.17939673364162445,
      "layer": 0.0879576038569212,
      "mlm": 1.5438344478607178,
      "distill": 0.9737842679023743,
      "teacher_acc": 0.718051016330719
    },
    {
      "step": 14741,
      "lr": 8.005163237769315e-07,
      "elapsed_s": 10950.079406738281,
      "loss": 2.7834248542785645,
      "hidden": 0.16955074667930603,
      "layer": 0.08497501909732819,
      "mlm": 1.4784665703773499,
      "distill": 0.9233692735433578,
      "teacher_acc": 0.727058157324791
    },
    {
      "step": 14751,
      "lr": 7.879270457969184e-07,
      "elapsed_s": 10957.475257873535,
      "loss": 2.7895267605781555,
      "hidden": 0.17260537296533585,
      "layer": 0.08561785519123077,
      "mlm": 1.466191291809082,
      "distill": 0.9353158622980118,
      "teacher_acc": 0.7240082770586014
    },
    {
      "step": 14761,
      "lr": 7.754359612344859e-07,
      "elapsed_s": 10964.890647172928,
      "loss": 2.8747113943099976,
      "hidden": 0.16749923676252365,
      "layer": 0.08415411971509457,
      "mlm": 1.5662117302417755,
      "distill": 0.9314241260290146,
      "teacher_acc": 0.6985058635473251
    },
    {
      "step": 14771,
      "lr": 7.630431207479277e-07,
      "elapsed_s": 10972.337652683258,
      "loss": 2.678408622741699,
      "hidden": 0.16675327718257904,
      "layer": 0.0834511611610651,
      "mlm": 1.4248882830142975,
      "distill": 0.8782881647348404,
      "teacher_acc": 0.7339321076869965
    },
    {
      "step": 14781,
      "lr": 7.507485745971043e-07,
      "elapsed_s": 10979.734361886978,
      "loss": 2.7640058994293213,
      "hidden": 0.17048689350485802,
      "layer": 0.08561656810343266,
      "mlm": 1.4236049354076385,
      "distill": 0.9566189497709274,
      "teacher_acc": 0.736858531832695
    },
    {
      "step": 14791,
      "lr": 7.385523726432364e-07,
      "elapsed_s": 10987.128791570663,
      "loss": 2.9336647987365723,
      "hidden": 0.1749587319791317,
      "layer": 0.08641347475349903,
      "mlm": 1.5253080427646637,
      "distill": 1.0152325183153152,
      "teacher_acc": 0.7199874073266983
    },
    {
      "step": 14801,
      "lr": 7.264545643486997e-07,
      "elapsed_s": 10994.506994485855,
      "loss": 2.9238389134407043,
      "hidden": 0.17551898211240768,
      "layer": 0.08702125027775764,
      "mlm": 1.5654364228248596,
      "distill": 0.9638539105653763,
      "teacher_acc": 0.7080443352460861
    },
    {
      "step": 14811,
      "lr": 7.144551987768338e-07,
      "elapsed_s": 11001.889144659042,
      "loss": 2.720135509967804,
      "hidden": 0.1703076921403408,
      "layer": 0.08404413796961308,
      "mlm": 1.4455814063549042,
      "distill": 0.8919166177511215,
      "teacher_acc": 0.7328233569860458
    },
    {
      "step": 14821,
      "lr": 7.02554324591742e-07,
      "elapsed_s": 11009.279310703278,
      "loss": 2.8809213638305664,
      "hidden": 0.17310554161667824,
      "layer": 0.08566122129559517,
      "mlm": 1.5834597647190094,
      "distill": 0.9084198921918869,
      "teacher_acc": 0.6890660971403122
    },
    {
      "step": 14831,
      "lr": 6.907519900580861e-07,
      "elapsed_s": 11016.695277452469,
      "loss": 2.84211403131485,
      "hidden": 0.17318176478147507,
      "layer": 0.0866678562015295,
      "mlm": 1.4875557124614716,
      "distill": 0.9648608565330505,
      "teacher_acc": 0.7249263375997543
    },
    {
      "step": 14841,
      "lr": 6.79048243040889e-07,
      "elapsed_s": 11024.071609973907,
      "loss": 2.8912243247032166,
      "hidden": 0.17129940167069435,
      "layer": 0.0856372807174921,
      "mlm": 1.577330619096756,
      "distill": 0.9284763187170029,
      "teacher_acc": 0.7010038197040558
    },
    {
      "step": 14851,
      "lr": 6.674431310053519e-07,
      "elapsed_s": 11031.486183643341,
      "loss": 2.6856426000595093,
      "hidden": 0.1736750639975071,
      "layer": 0.08531162142753601,
      "mlm": 1.3865609765052795,
      "distill": 0.9090756177902222,
      "teacher_acc": 0.7390550673007965
    },
    {
      "step": 14861,
      "lr": 6.559367010166628e-07,
      "elapsed_s": 11038.906079530716,
      "loss": 2.8512858152389526,
      "hidden": 0.17514295130968094,
      "layer": 0.08675952069461346,
      "mlm": 1.4962226450443268,
      "distill": 0.9613975286483765,
      "teacher_acc": 0.7240743786096573
    },
    {
      "step": 14871,
      "lr": 6.44528999739788e-07,
      "elapsed_s": 11046.336796283722,
      "loss": 2.6999950408935547,
      "hidden": 0.1696496084332466,
      "layer": 0.08490796200931072,
      "mlm": 1.4013757407665253,
      "distill": 0.9168660938739777,
      "teacher_acc": 0.7382998615503311
    },
    {
      "step": 14881,
      "lr": 6.332200734393057e-07,
      "elapsed_s": 11053.765421390533,
      "loss": 2.891092538833618,
      "hidden": 0.17457423359155655,
      "layer": 0.08579068444669247,
      "mlm": 1.5424642860889435,
      "distill": 0.9565844386816025,
      "teacher_acc": 0.7157268971204758
    },
    {
      "step": 14891,
      "lr": 6.220099679791897e-07,
      "elapsed_s": 11061.224831104279,
      "loss": 2.87156218290329,
      "hidden": 0.17565599083900452,
      "layer": 0.08607467077672482,
      "mlm": 1.5146782398223877,
      "distill": 0.962534710764885,
      "teacher_acc": 0.7170919477939606
    },
    {
      "step": 14901,
      "lr": 6.108987288226536e-07,
      "elapsed_s": 11068.579588413239,
      "loss": 2.8931609988212585,
      "hidden": 0.17428219318389893,
      "layer": 0.08652433939278126,
      "mlm": 1.5220081806182861,
      "distill": 0.9793262332677841,
      "teacher_acc": 0.7207663655281067
    },
    {
      "step": 14911,
      "lr": 5.998864010319516e-07,
      "elapsed_s": 11076.005879878998,
      "loss": 2.8689403533935547,
      "hidden": 0.16883435100317,
      "layer": 0.08595230989158154,
      "mlm": 1.5905997157096863,
      "distill": 0.8976957648992538,
      "teacher_acc": 0.6956483274698257
    },
    {
      "step": 14921,
      "lr": 5.889730292681972e-07,
      "elapsed_s": 11083.402671813965,
      "loss": 2.782044768333435,
      "hidden": 0.17074234038591385,
      "layer": 0.0857284776866436,
      "mlm": 1.434402346611023,
      "distill": 0.9632934033870697,
      "teacher_acc": 0.7369923889636993
    },
    {
      "step": 14931,
      "lr": 5.781586577911807e-07,
      "elapsed_s": 11090.83666753769,
      "loss": 2.8406795263290405,
      "hidden": 0.17498242855072021,
      "layer": 0.08501601405441761,
      "mlm": 1.5035149157047272,
      "distill": 0.9446917176246643,
      "teacher_acc": 0.7136219292879105
    },
    {
      "step": 14941,
      "lr": 5.674433304591859e-07,
      "elapsed_s": 11098.282428503036,
      "loss": 2.9502903819084167,
      "hidden": 0.17157406359910965,
      "layer": 0.08604967035353184,
      "mlm": 1.6335065960884094,
      "distill": 0.930610716342926,
      "teacher_acc": 0.688972994685173
    },
    {
      "step": 14951,
      "lr": 5.568270907288287e-07,
      "elapsed_s": 11105.713072538376,
      "loss": 3.057758152484894,
      "hidden": 0.182733204215765,
      "layer": 0.08912954479455948,
      "mlm": 1.613631159067154,
      "distill": 1.0340958386659622,
      "teacher_acc": 0.6886073648929596
    },
    {
      "step": 14961,
      "lr": 5.463099816548579e-07,
      "elapsed_s": 11113.147362470627,
      "loss": 2.838450849056244,
      "hidden": 0.1726633571088314,
      "layer": 0.08526210114359856,
      "mlm": 1.5155292451381683,
      "distill": 0.9349637925624847,
      "teacher_acc": 0.7131546437740326
    },
    {
      "step": 14971,
      "lr": 5.358920458900046e-07,
      "elapsed_s": 11120.59383058548,
      "loss": 2.7409510016441345,
      "hidden": 0.1745554320514202,
      "layer": 0.08602048270404339,
      "mlm": 1.3818857967853546,
      "distill": 0.9669440388679504,
      "teacher_acc": 0.7473931163549423
    },
    {
      "step": 14981,
      "lr": 5.255733256847889e-07,
      "elapsed_s": 11128.035746097565,
      "loss": 2.888481855392456,
      "hidden": 0.17650017514824867,
      "layer": 0.08692662604153156,
      "mlm": 1.4895232021808624,
      "distill": 1.0024949759244919,
      "teacher_acc": 0.7313336133956909
    },
    {
      "step": 14991,
      "lr": 5.15353862887355e-07,
      "elapsed_s": 11135.480044603348,
      "loss": 2.767621636390686,
      "hidden": 0.17124944925308228,
      "layer": 0.08611654862761497,
      "mlm": 1.449194997549057,
      "distill": 0.9328694343566895,
      "teacher_acc": 0.7302902787923813
    },
    {
      "step": 15001,
      "lr": 5.052336989433082e-07,
      "elapsed_s": 11144.962507009506,
      "loss": 2.7852954268455505,
      "hidden": 0.16580792516469955,
      "layer": 0.08434540964663029,
      "mlm": 1.5190192759037018,
      "distill": 0.8924875557422638,
      "teacher_acc": 0.7058766037225723
    },
    {
      "step": 15011,
      "lr": 4.952128748955398e-07,
      "elapsed_s": 11152.387697935104,
      "loss": 2.8101967573165894,
      "hidden": 0.1703622080385685,
      "layer": 0.08521559461951256,
      "mlm": 1.5014455020427704,
      "distill": 0.9254190027713776,
      "teacher_acc": 0.7126597315073013
    },
    {
      "step": 15021,
      "lr": 4.852914313840662e-07,
      "elapsed_s": 11159.805647611618,
      "loss": 2.892549514770508,
      "hidden": 0.17474956065416336,
      "layer": 0.08611348643898964,
      "mlm": 1.5126395225524902,
      "distill": 0.9873540997505188,
      "teacher_acc": 0.7213145792484283
    },
    {
      "step": 15031,
      "lr": 4.7546940864585387e-07,
      "elapsed_s": 11167.245911121368,
      "loss": 2.7559183835983276,
      "hidden": 0.1737385205924511,
      "layer": 0.08533759973943233,
      "mlm": 1.4436572194099426,
      "distill": 0.922115296125412,
      "teacher_acc": 0.7263970077037811
    },
    {
      "step": 15041,
      "lr": 4.6574684651466415e-07,
      "elapsed_s": 11174.70185470581,
      "loss": 2.8239579796791077,
      "hidden": 0.1735365055501461,
      "layer": 0.08519955538213253,
      "mlm": 1.4950356483459473,
      "distill": 0.939249575138092,
      "teacher_acc": 0.7233349680900574
    },
    {
      "step": 15051,
      "lr": 4.5612378442089766e-07,
      "elapsed_s": 11182.15726184845,
      "loss": 2.887046158313751,
      "hidden": 0.1757766604423523,
      "layer": 0.08700550347566605,
      "mlm": 1.5277834832668304,
      "distill": 0.964206650853157,
      "teacher_acc": 0.7099229246377945
    },
    {
      "step": 15061,
      "lr": 4.4660026139141385e-07,
      "elapsed_s": 11189.59115767479,
      "loss": 2.9336275458335876,
      "hidden": 0.17395057901740074,
      "layer": 0.08630500175058842,
      "mlm": 1.574600338935852,
      "distill": 0.9679735898971558,
      "teacher_acc": 0.6972937881946564
    },
    {
      "step": 15071,
      "lr": 4.371763160494008e-07,
      "elapsed_s": 11197.014613389969,
      "loss": 2.673524022102356,
      "hidden": 0.1666432023048401,
      "layer": 0.0833622869104147,
      "mlm": 1.4084221720695496,
      "distill": 0.890134260058403,
      "teacher_acc": 0.7342276126146317
    },
    {
      "step": 15081,
      "lr": 4.278519866141972e-07,
      "elapsed_s": 11204.436433315277,
      "loss": 2.7788753509521484,
      "hidden": 0.16438215598464012,
      "layer": 0.08395221643149853,
      "mlm": 1.5118547081947327,
      "distill": 0.8962802439928055,
      "teacher_acc": 0.7074703872203827
    },
    {
      "step": 15091,
      "lr": 4.1862731090113736e-07,
      "elapsed_s": 11211.850955963135,
      "loss": 2.8283971548080444,
      "hidden": 0.16993208229541779,
      "layer": 0.08386317826807499,
      "mlm": 1.5140541195869446,
      "distill": 0.9325472712516785,
      "teacher_acc": 0.71501125395298
    },
    {
      "step": 15101,
      "lr": 4.095023263214121e-07,
      "elapsed_s": 11219.232813358307,
      "loss": 2.7722848057746887,
      "hidden": 0.17030474543571472,
      "layer": 0.08593831025063992,
      "mlm": 1.4777165353298187,
      "distill": 0.9109897166490555,
      "teacher_acc": 0.723077118396759
    },
    {
      "step": 15111,
      "lr": 4.004770698819077e-07,
      "elapsed_s": 11226.648528575897,
      "loss": 2.8092472553253174,
      "hidden": 0.17700574919581413,
      "layer": 0.08727842383086681,
      "mlm": 1.4097817540168762,
      "distill": 1.00181482732296,
      "teacher_acc": 0.7426606565713882
    },
    {
      "step": 15121,
      "lr": 3.915515781850565e-07,
      "elapsed_s": 11234.046501874924,
      "loss": 2.775519073009491,
      "hidden": 0.16705475002527237,
      "layer": 0.08439456671476364,
      "mlm": 1.475652813911438,
      "distill": 0.9235594868659973,
      "teacher_acc": 0.7204635739326477
    },
    {
      "step": 15131,
      "lr": 3.827258874286921e-07,
      "elapsed_s": 11241.43199634552,
      "loss": 2.8294626474380493,
      "hidden": 0.170627199113369,
      "layer": 0.08523484319448471,
      "mlm": 1.487058013677597,
      "distill": 0.9585328251123428,
      "teacher_acc": 0.7237738072872162
    },
    {
      "step": 15141,
      "lr": 3.7400003340589687e-07,
      "elapsed_s": 11248.831955432892,
      "loss": 2.7984516620635986,
      "hidden": 0.17136017233133316,
      "layer": 0.08504154719412327,
      "mlm": 1.5199095606803894,
      "distill": 0.8933009952306747,
      "teacher_acc": 0.711412638425827
    },
    {
      "step": 15151,
      "lr": 3.653740515048576e-07,
      "elapsed_s": 11256.241410255432,
      "loss": 2.740207076072693,
      "hidden": 0.1714858077466488,
      "layer": 0.08514353260397911,
      "mlm": 1.4663298726081848,
      "distill": 0.888333797454834,
      "teacher_acc": 0.7218104004859924
    },
    {
      "step": 15161,
      "lr": 3.568479767087296e-07,
      "elapsed_s": 11263.62489104271,
      "loss": 2.8266592621803284,
      "hidden": 0.1737096905708313,
      "layer": 0.08576507121324539,
      "mlm": 1.474443793296814,
      "distill": 0.9619135558605194,
      "teacher_acc": 0.7256395667791367
    },
    {
      "step": 15171,
      "lr": 3.484218435954839e-07,
      "elapsed_s": 11271.041904449463,
      "loss": 2.7838099598884583,
      "hidden": 0.16890650242567062,
      "layer": 0.08483339846134186,
      "mlm": 1.5078443884849548,
      "distill": 0.8957358300685883,
      "teacher_acc": 0.7034810483455658
    },
    {
      "step": 15181,
      "lr": 3.4009568633777964e-07,
      "elapsed_s": 11278.43856716156,
      "loss": 2.783474326133728,
      "hidden": 0.17270272970199585,
      "layer": 0.08548083528876305,
      "mlm": 1.4582253098487854,
      "distill": 0.9371031671762466,
      "teacher_acc": 0.7310761660337448
    },
    {
      "step": 15191,
      "lr": 3.3186953870281144e-07,
      "elapsed_s": 11285.848642349243,
      "loss": 2.724146842956543,
      "hidden": 0.174867145717144,
      "layer": 0.08671503514051437,
      "mlm": 1.4237720966339111,
      "distill": 0.9072829335927963,
      "teacher_acc": 0.7289159595966339
    },
    {
      "step": 15201,
      "lr": 3.237434340521789e-07,
      "elapsed_s": 11293.2278444767,
      "loss": 2.842944622039795,
      "hidden": 0.17046865075826645,
      "layer": 0.08616664819419384,
      "mlm": 1.4970521032810211,
      "distill": 0.9618718475103378,
      "teacher_acc": 0.7204853147268295
    },
    {
      "step": 15211,
      "lr": 3.157174053417589e-07,
      "elapsed_s": 11300.616320848465,
      "loss": 2.6735154390335083,
      "hidden": 0.16860781610012054,
      "layer": 0.0845844317227602,
      "mlm": 1.3955365121364594,
      "distill": 0.8984711468219757,
      "teacher_acc": 0.7283686846494675
    },
    {
      "step": 15221,
      "lr": 3.077914851215585e-07,
      "elapsed_s": 11307.995571613312,
      "loss": 2.820692777633667,
      "hidden": 0.17071133479475975,
      "layer": 0.0852662306278944,
      "mlm": 1.5015047192573547,
      "distill": 0.9351322650909424,
      "teacher_acc": 0.7226359099149704
    },
    {
      "step": 15231,
      "lr": 2.9996570553559036e-07,
      "elapsed_s": 11315.422125816345,
      "loss": 2.76784211397171,
      "hidden": 0.17098674550652504,
      "layer": 0.08554156310856342,
      "mlm": 1.4920405745506287,
      "distill": 0.8910572379827499,
      "teacher_acc": 0.7135284394025803
    },
    {
      "step": 15241,
      "lr": 2.922400983217416e-07,
      "elapsed_s": 11322.840555429459,
      "loss": 2.828190326690674,
      "hidden": 0.1737024113535881,
      "layer": 0.08665317110717297,
      "mlm": 1.501670002937317,
      "distill": 0.9357889145612717,
      "teacher_acc": 0.7138854116201401
    },
    {
      "step": 15251,
      "lr": 2.846146948116468e-07,
      "elapsed_s": 11330.231798648834,
      "loss": 2.847204625606537,
      "hidden": 0.17968131601810455,
      "layer": 0.08791285008192062,
      "mlm": 1.438569724559784,
      "distill": 1.0053158104419708,
      "teacher_acc": 0.7285197228193283
    },
    {
      "step": 15261,
      "lr": 2.770895259305517e-07,
      "elapsed_s": 11337.652816295624,
      "loss": 2.7813690900802612,
      "hidden": 0.17074116319417953,
      "layer": 0.0855549443513155,
      "mlm": 1.4502010941505432,
      "distill": 0.9469082355499268,
      "teacher_acc": 0.7345093339681625
    },
    {
      "step": 15271,
      "lr": 2.6966462219720767e-07,
      "elapsed_s": 11345.087265968323,
      "loss": 2.8319063782691956,
      "hidden": 0.17446884140372276,
      "layer": 0.08587104827165604,
      "mlm": 1.5276874601840973,
      "distill": 0.9123456478118896,
      "teacher_acc": 0.706764742732048
    },
    {
      "step": 15281,
      "lr": 2.6234001372372194e-07,
      "elapsed_s": 11352.514022827148,
      "loss": 2.9520995020866394,
      "hidden": 0.17718017101287842,
      "layer": 0.08712559193372726,
      "mlm": 1.549910694360733,
      "distill": 1.0042656362056732,
      "teacher_acc": 0.7122835069894791
    },
    {
      "step": 15291,
      "lr": 2.5511573021545775e-07,
      "elapsed_s": 11359.91390490532,
      "loss": 2.9148569107055664,
      "hidden": 0.1740666702389717,
      "layer": 0.08655950240790844,
      "mlm": 1.544133722782135,
      "distill": 0.9793101400136948,
      "teacher_acc": 0.7155590057373047
    },
    {
      "step": 15301,
      "lr": 2.479918009708981e-07,
      "elapsed_s": 11367.32721042633,
      "loss": 2.8702393770217896,
      "hidden": 0.1685887724161148,
      "layer": 0.085444500669837,
      "mlm": 1.5528983175754547,
      "distill": 0.937441274523735,
      "teacher_acc": 0.7047758102416992
    },
    {
      "step": 15311,
      "lr": 2.4096825488154063e-07,
      "elapsed_s": 11374.730768680573,
      "loss": 2.734600245952606,
      "hidden": 0.1728328838944435,
      "layer": 0.08564070425927639,
      "mlm": 1.421353965997696,
      "distill": 0.9247601479291916,
      "teacher_acc": 0.7346855700016022
    },
    {
      "step": 15321,
      "lr": 2.3404512043176675e-07,
      "elapsed_s": 11382.143733739853,
      "loss": 2.7793914675712585,
      "hidden": 0.17512977868318558,
      "layer": 0.08723573945462704,
      "mlm": 1.4644586443901062,
      "distill": 0.9210553616285324,
      "teacher_acc": 0.726467490196228
    },
    {
      "step": 15331,
      "lr": 2.2722242569873654e-07,
      "elapsed_s": 11389.524953126907,
      "loss": 2.761188566684723,
      "hidden": 0.17053695395588875,
      "layer": 0.08595173247158527,
      "mlm": 1.4431284070014954,
      "distill": 0.9340104162693024,
      "teacher_acc": 0.7336760312318802
    },
    {
      "step": 15341,
      "lr": 2.2050019835227198e-07,
      "elapsed_s": 11396.910083055496,
      "loss": 2.674768567085266,
      "hidden": 0.16720469668507576,
      "layer": 0.0843095351010561,
      "mlm": 1.3607600033283234,
      "distill": 0.9374445080757141,
      "teacher_acc": 0.7474536150693893
    },
    {
      "step": 15351,
      "lr": 2.1387846565474045e-07,
      "elapsed_s": 11404.321236371994,
      "loss": 2.7620485424995422,
      "hidden": 0.17009568959474564,
      "layer": 0.08555116504430771,
      "mlm": 1.453881412744522,
      "distill": 0.9252002388238907,
      "teacher_acc": 0.7210217416286469
    },
    {
      "step": 15361,
      "lr": 2.0735725446094923e-07,
      "elapsed_s": 11411.71713423729,
      "loss": 2.8076502084732056,
      "hidden": 0.17113498225808144,
      "layer": 0.08519008755683899,
      "mlm": 1.4688905775547028,
      "distill": 0.9538945704698563,
      "teacher_acc": 0.7295829802751541
    },
    {
      "step": 15371,
      "lr": 2.009365912180372e-07,
      "elapsed_s": 11419.124678134918,
      "loss": 2.7209373712539673,
      "hidden": 0.16764381155371666,
      "layer": 0.08463571406900883,
      "mlm": 1.4407899975776672,
      "distill": 0.9025418609380722,
      "teacher_acc": 0.7254003584384918
    },
    {
      "step": 15381,
      "lr": 1.9461650196536675e-07,
      "elapsed_s": 11426.508699655533,
      "loss": 2.8376436829566956,
      "hidden": 0.17487308755517006,
      "layer": 0.08794470317661762,
      "mlm": 1.433811217546463,
      "distill": 1.0101139545440674,
      "teacher_acc": 0.738265261054039
    },
    {
      "step": 15391,
      "lr": 1.8839701233440697e-07,
      "elapsed_s": 11433.90235209465,
      "loss": 2.7133408784866333,
      "hidden": 0.174795251339674,
      "layer": 0.0857569258660078,
      "mlm": 1.3814596831798553,
      "distill": 0.9394122213125229,
      "teacher_acc": 0.7513058185577393
    },
    {
      "step": 15401,
      "lr": 1.8227814754865068e-07,
      "elapsed_s": 11441.23759508133,
      "loss": 2.930616557598114,
      "hidden": 0.17531239613890648,
      "layer": 0.086771784350276,
      "mlm": 1.5424167215824127,
      "distill": 0.9941891133785248,
      "teacher_acc": 0.7146186083555222
    },
    {
      "step": 15411,
      "lr": 1.762599324234948e-07,
      "elapsed_s": 11448.62261557579,
      "loss": 2.866721212863922,
      "hidden": 0.17771394923329353,
      "layer": 0.08792668022215366,
      "mlm": 1.4676828384399414,
      "distill": 0.9996471405029297,
      "teacher_acc": 0.7261768132448196
    },
    {
      "step": 15421,
      "lr": 1.7034239136614894e-07,
      "elapsed_s": 11456.047062635422,
      "loss": 2.8820990324020386,
      "hidden": 0.17983843758702278,
      "layer": 0.08759019523859024,
      "mlm": 1.4681333005428314,
      "distill": 1.0104937106370926,
      "teacher_acc": 0.7319972813129425
    },
    {
      "step": 15431,
      "lr": 1.645255483755326e-07,
      "elapsed_s": 11463.449441194534,
      "loss": 2.8942481875419617,
      "hidden": 0.17976617068052292,
      "layer": 0.08776849322021008,
      "mlm": 1.4939399361610413,
      "distill": 0.996891662478447,
      "teacher_acc": 0.7243596911430359
    },
    {
      "step": 15441,
      "lr": 1.5880942704217527e-07,
      "elapsed_s": 11470.891865730286,
      "loss": 2.8291927576065063,
      "hidden": 0.17578815668821335,
      "layer": 0.08656280860304832,
      "mlm": 1.460192620754242,
      "distill": 0.9741424471139908,
      "teacher_acc": 0.721147283911705
    },
    {
      "step": 15451,
      "lr": 1.5319405054813052e-07,
      "elapsed_s": 11478.332643508911,
      "loss": 2.7968022227287292,
      "hidden": 0.17093294113874435,
      "layer": 0.08572666347026825,
      "mlm": 1.4891286492347717,
      "distill": 0.9229443371295929,
      "teacher_acc": 0.7256985157728195
    },
    {
      "step": 15461,
      "lr": 1.476794416668703e-07,
      "elapsed_s": 11485.751631259918,
      "loss": 2.900018632411957,
      "hidden": 0.1731034368276596,
      "layer": 0.08644472435116768,
      "mlm": 1.5199704766273499,
      "distill": 0.9906189292669296,
      "teacher_acc": 0.7156846523284912
    },
    {
      "step": 15471,
      "lr": 1.422656227631991e-07,
      "elapsed_s": 11493.179045677185,
      "loss": 2.74891996383667,
      "hidden": 0.17418503016233444,
      "layer": 0.08691488020122051,
      "mlm": 1.4397330284118652,
      "distill": 0.9173594266176224,
      "teacher_acc": 0.7351050227880478
    },
    {
      "step": 15481,
      "lr": 1.3695261579316777e-07,
      "elapsed_s": 11500.58942270279,
      "loss": 2.797867000102997,
      "hidden": 0.187094334512949,
      "layer": 0.09093068540096283,
      "mlm": 1.3367797434329987,
      "distill": 1.0414333194494247,
      "teacher_acc": 0.7683060169219971
    },
    {
      "step": 15491,
      "lr": 1.3174044230396532e-07,
      "elapsed_s": 11507.972880601883,
      "loss": 2.697475790977478,
      "hidden": 0.16979219019412994,
      "layer": 0.08494482561945915,
      "mlm": 1.4055123627185822,
      "distill": 0.9099066704511642,
      "teacher_acc": 0.7353493720293045
    },
    {
      "step": 15501,
      "lr": 1.266291234338607e-07,
      "elapsed_s": 11515.363358259201,
      "loss": 2.774468421936035,
      "hidden": 0.17112699896097183,
      "layer": 0.08599593490362167,
      "mlm": 1.4664761126041412,
      "distill": 0.9227403551340103,
      "teacher_acc": 0.7165646702051163
    },
    {
      "step": 15511,
      "lr": 1.2161867991208886e-07,
      "elapsed_s": 11522.779840707779,
      "loss": 2.719054102897644,
      "hidden": 0.1716201901435852,
      "layer": 0.0852173175662756,
      "mlm": 1.442199170589447,
      "distill": 0.8910059183835983,
      "teacher_acc": 0.729556068778038
    },
    {
      "step": 15521,
      "lr": 1.167091320587843e-07,
      "elapsed_s": 11530.179162740707,
      "loss": 2.6373308897018433,
      "hidden": 0.16968093067407608,
      "layer": 0.08435825631022453,
      "mlm": 1.3382934629917145,
      "distill": 0.9174964725971222,
      "teacher_acc": 0.7584157586097717
    },
    {
      "step": 15531,
      "lr": 1.119004997848977e-07,
      "elapsed_s": 11537.605904340744,
      "loss": 2.8385590314865112,
      "hidden": 0.17810656130313873,
      "layer": 0.08805380575358868,
      "mlm": 1.445037990808487,
      "distill": 0.9932809770107269,
      "teacher_acc": 0.7315360903739929
    },
    {
      "step": 15541,
      "lr": 1.0719280259210152e-07,
      "elapsed_s": 11545.037128925323,
      "loss": 2.856227397918701,
      "hidden": 0.1728021465241909,
      "layer": 0.08584113605320454,
      "mlm": 1.45015949010849,
      "distill": 1.0175431072711945,
      "teacher_acc": 0.7391324192285538
    },
    {
      "step": 15551,
      "lr": 1.0258605957272627e-07,
      "elapsed_s": 11552.449007749557,
      "loss": 2.767713725566864,
      "hidden": 0.17111529409885406,
      "layer": 0.08525202050805092,
      "mlm": 1.4145541191101074,
      "distill": 0.9683030396699905,
      "teacher_acc": 0.7463905513286591
    },
    {
      "step": 15561,
      "lr": 9.808028940967718e-08,
      "elapsed_s": 11559.867485761642,
      "loss": 2.712779223918915,
      "hidden": 0.16991673409938812,
      "layer": 0.08541332371532917,
      "mlm": 1.4094305634498596,
      "distill": 0.9208085387945175,
      "teacher_acc": 0.7332517504692078
    },
    {
      "step": 15571,
      "lr": 9.367551037635924e-08,
      "elapsed_s": 11567.29406094551,
      "loss": 2.781728982925415,
      "hidden": 0.16719579324126244,
      "layer": 0.08423412591218948,
      "mlm": 1.4767448604106903,
      "distill": 0.9284754991531372,
      "teacher_acc": 0.7210367023944855
    },
    {
      "step": 15581,
      "lr": 8.937174033659956e-08,
      "elapsed_s": 11574.731894731522,
      "loss": 2.6527915596961975,
      "hidden": 0.16574443131685257,
      "layer": 0.08442026749253273,
      "mlm": 1.38564932346344,
      "distill": 0.893443301320076,
      "teacher_acc": 0.7301554828882217
    },
    {
      "step": 15591,
      "lr": 8.516899674457513e-08,
      "elapsed_s": 11582.152236700058,
      "loss": 2.731054425239563,
      "hidden": 0.16816063970327377,
      "layer": 0.08369730040431023,
      "mlm": 1.5257735848426819,
      "distill": 0.8271108567714691,
      "teacher_acc": 0.6990182250738144
    },
    {
      "step": 15601,
      "lr": 8.106729664475176e-08,
      "elapsed_s": 11589.59359741211,
      "loss": 2.7752684950828552,
      "hidden": 0.16980299726128578,
      "layer": 0.08391544967889786,
      "mlm": 1.4720353186130524,
      "distill": 0.9216694384813309,
      "teacher_acc": 0.7206307947635651
    },
    {
      "step": 15611,
      "lr": 7.706665667180091e-08,
      "elapsed_s": 11597.043562173843,
      "loss": 2.866943836212158,
      "hidden": 0.17594660818576813,
      "layer": 0.08696525916457176,
      "mlm": 1.4615337252616882,
      "distill": 1.0100342780351639,
      "teacher_acc": 0.7308745086193085
    },
    {
      "step": 15621,
      "lr": 7.316709305054403e-08,
      "elapsed_s": 11604.479805231094,
      "loss": 2.8374080061912537,
      "hidden": 0.17499275505542755,
      "layer": 0.08651051856577396,
      "mlm": 1.478503704071045,
      "distill": 0.9656636118888855,
      "teacher_acc": 0.7266622483730316
    },
    {
      "step": 15631,
      "lr": 6.936862159588054e-08,
      "elapsed_s": 11611.885043621063,
      "loss": 2.7846860885620117,
      "hidden": 0.17538509890437126,
      "layer": 0.08712207712233067,
      "mlm": 1.4803510308265686,
      "distill": 0.910003736615181,
      "teacher_acc": 0.7198794931173325
    },
    {
      "step": 15641,
      "lr": 6.567125771272387e-08,
      "elapsed_s": 11619.297068119049,
      "loss": 2.7804161310195923,
      "hidden": 0.16963930428028107,
      "layer": 0.08610764518380165,
      "mlm": 1.481711745262146,
      "distill": 0.9163719862699509,
      "teacher_acc": 0.7207148373126984
    },
    {
      "step": 15651,
      "lr": 6.207501639593493e-08,
      "elapsed_s": 11626.726328372955,
      "loss": 2.7207136750221252,
      "hidden": 0.17279021069407463,
      "layer": 0.08498672768473625,
      "mlm": 1.3902694880962372,
      "distill": 0.9423703998327255,
      "teacher_acc": 0.7362833768129349
    },
    {
      "step": 15661,
      "lr": 5.8579912230280476e-08,
      "elapsed_s": 11634.180311441422,
      "loss": 2.8041076064109802,
      "hidden": 0.1762543022632599,
      "layer": 0.0865848958492279,
      "mlm": 1.4167143404483795,
      "distill": 0.991592213511467,
      "teacher_acc": 0.746922492980957
    },
    {
      "step": 15671,
      "lr": 5.5185959390344214e-08,
      "elapsed_s": 11641.609824895859,
      "loss": 2.8151450753211975,
      "hidden": 0.17407775297760963,
      "layer": 0.08576864935457706,
      "mlm": 1.4950434863567352,
      "distill": 0.929061770439148,
      "teacher_acc": 0.715582087635994
    },
    {
      "step": 15681,
      "lr": 5.189317164049634e-08,
      "elapsed_s": 11649.019064188004,
      "loss": 2.709412634372711,
      "hidden": 0.17108751833438873,
      "layer": 0.08563871309161186,
      "mlm": 1.3972118198871613,
      "distill": 0.927206352353096,
      "teacher_acc": 0.7404645830392838
    },
    {
      "step": 15691,
      "lr": 4.87015623348186e-08,
      "elapsed_s": 11656.459064245224,
      "loss": 2.766219973564148,
      "hidden": 0.17129210010170937,
      "layer": 0.08537965640425682,
      "mlm": 1.4616104066371918,
      "distill": 0.9193355590105057,
      "teacher_acc": 0.7261091768741608
    },
    {
      "step": 15701,
      "lr": 4.56111444170626e-08,
      "elapsed_s": 11663.897270202637,
      "loss": 2.7890841364860535,
      "hidden": 0.17125079780817032,
      "layer": 0.08567921072244644,
      "mlm": 1.459015130996704,
      "distill": 0.9447278380393982,
      "teacher_acc": 0.7240240126848221
    },
    {
      "step": 15711,
      "lr": 4.262193042059437e-08,
      "elapsed_s": 11671.331955194473,
      "loss": 2.8594861030578613,
      "hidden": 0.17046421393752098,
      "layer": 0.08591949567198753,
      "mlm": 1.5165071785449982,
      "distill": 0.9590907394886017,
      "teacher_acc": 0.7138648182153702
    },
    {
      "step": 15721,
      "lr": 3.9733932468333234e-08,
      "elapsed_s": 11678.756972551346,
      "loss": 2.7832525968551636,
      "hidden": 0.16877593100070953,
      "layer": 0.08500281721353531,
      "mlm": 1.4773665368556976,
      "distill": 0.9258327484130859,
      "teacher_acc": 0.722317099571228
    },
    {
      "step": 15731,
      "lr": 3.694716227271855e-08,
      "elapsed_s": 11686.15344619751,
      "loss": 2.7412885427474976,
      "hidden": 0.16807739436626434,
      "layer": 0.08430268615484238,
      "mlm": 1.4419969022274017,
      "distill": 0.920985534787178,
      "teacher_acc": 0.7319333702325821
    },
    {
      "step": 15741,
      "lr": 3.426163113565417e-08,
      "elapsed_s": 11693.575344324112,
      "loss": 2.8108537793159485,
      "hidden": 0.1713666431605816,
      "layer": 0.086165651679039,
      "mlm": 1.4792713522911072,
      "distill": 0.945766344666481,
      "teacher_acc": 0.720917820930481
    },
    {
      "step": 15751,
      "lr": 3.167734994846128e-08,
      "elapsed_s": 11700.987768888474,
      "loss": 2.872600495815277,
      "hidden": 0.17223602533340454,
      "layer": 0.0851020161062479,
      "mlm": 1.5030353963375092,
      "distill": 0.9825419932603836,
      "teacher_acc": 0.7255667448043823
    },
    {
      "step": 15761,
      "lr": 2.919432919183396e-08,
      "elapsed_s": 11708.395985126495,
      "loss": 2.9179238080978394,
      "hidden": 0.17500020563602448,
      "layer": 0.08687887154519558,
      "mlm": 1.5587475001811981,
      "distill": 0.9657364785671234,
      "teacher_acc": 0.6993413120508194
    },
    {
      "step": 15771,
      "lr": 2.6812578935808685e-08,
      "elapsed_s": 11715.799767971039,
      "loss": 2.8110761046409607,
      "hidden": 0.17201600223779678,
      "layer": 0.08508848026394844,
      "mlm": 1.5100273191928864,
      "distill": 0.9144725948572159,
      "teacher_acc": 0.713136151432991
    },
    {
      "step": 15781,
      "lr": 2.453210883970325e-08,
      "elapsed_s": 11723.212426662445,
      "loss": 2.9278329014778137,
      "hidden": 0.17803650349378586,
      "layer": 0.08790086023509502,
      "mlm": 1.4933663308620453,
      "distill": 1.034443125128746,
      "teacher_acc": 0.7301563024520874
    },
    {
      "step": 15791,
      "lr": 2.2352928152089003e-08,
      "elapsed_s": 11730.609036445618,
      "loss": 2.749425530433655,
      "hidden": 0.16751904040575027,
      "layer": 0.08425705507397652,
      "mlm": 1.449153572320938,
      "distill": 0.9231053739786148,
      "teacher_acc": 0.723153829574585
    },
    {
      "step": 15801,
      "lr": 2.0275045710760334e-08,
      "elapsed_s": 11737.999054670334,
      "loss": 2.7756863236427307,
      "hidden": 0.17044877633452415,
      "layer": 0.0850465577095747,
      "mlm": 1.4676402509212494,
      "distill": 0.9246252477169037,
      "teacher_acc": 0.7168364822864532
    },
    {
      "step": 15811,
      "lr": 1.8298469942681916e-08,
      "elapsed_s": 11745.423281908035,
      "loss": 2.8402692675590515,
      "hidden": 0.17299408465623856,
      "layer": 0.0863008163869381,
      "mlm": 1.5062010288238525,
      "distill": 0.9449296444654465,
      "teacher_acc": 0.7153492122888565
    },
    {
      "step": 15821,
      "lr": 1.642320886396931e-08,
      "elapsed_s": 11752.82661986351,
      "loss": 2.7874521613121033,
      "hidden": 0.17109298706054688,
      "layer": 0.08533688075840473,
      "mlm": 1.452109545469284,
      "distill": 0.9504881948232651,
      "teacher_acc": 0.7301358282566071
    },
    {
      "step": 15831,
      "lr": 1.4649270079847289e-08,
      "elapsed_s": 11760.205814123154,
      "loss": 2.7630205154418945,
      "hidden": 0.16908187419176102,
      "layer": 0.08499930985271931,
      "mlm": 1.4343226552009583,
      "distill": 0.9480345249176025,
      "teacher_acc": 0.7382134795188904
    },
    {
      "step": 15841,
      "lr": 1.297666078462767e-08,
      "elapsed_s": 11767.572574138641,
      "loss": 2.832963705062866,
      "hidden": 0.17833144962787628,
      "layer": 0.08663557656109333,
      "mlm": 1.46321240067482,
      "distill": 0.9697706550359726,
      "teacher_acc": 0.7351847290992737
    },
    {
      "step": 15851,
      "lr": 1.1405387761664887e-08,
      "elapsed_s": 11774.998354911804,
      "loss": 2.851606845855713,
      "hidden": 0.17633429169654846,
      "layer": 0.08667857572436333,
      "mlm": 1.4800944030284882,
      "distill": 0.9755045622587204,
      "teacher_acc": 0.7151873707771301
    },
    {
      "step": 15861,
      "lr": 9.935457383350444e-09,
      "elapsed_s": 11782.42622590065,
      "loss": 2.697765529155731,
      "hidden": 0.16493499279022217,
      "layer": 0.0839875377714634,
      "mlm": 1.4364041984081268,
      "distill": 0.8894975334405899,
      "teacher_acc": 0.7233511954545975
    },
    {
      "step": 15871,
      "lr": 8.566875611068504e-09,
      "elapsed_s": 11789.846040725708,
      "loss": 2.838391423225403,
      "hidden": 0.1720663458108902,
      "layer": 0.08621477521955967,
      "mlm": 1.5018529891967773,
      "distill": 0.9492983669042587,
      "teacher_acc": 0.7157105356454849
    },
    {
      "step": 15881,
      "lr": 7.299647995176462e-09,
      "elapsed_s": 11797.272265195847,
      "loss": 2.691572427749634,
      "hidden": 0.16422618553042412,
      "layer": 0.08310709707438946,
      "mlm": 1.4307821691036224,
      "distill": 0.890784278512001,
      "teacher_acc": 0.7298342734575272
    },
    {
      "step": 15891,
      "lr": 6.133779674988294e-09,
      "elapsed_s": 11804.711457014084,
      "loss": 2.728629410266876,
      "hidden": 0.16765647381544113,
      "layer": 0.08523947186768055,
      "mlm": 1.409932404756546,
      "distill": 0.9407642930746078,
      "teacher_acc": 0.7432170361280441
    },
    {
      "step": 15901,
      "lr": 5.069275378746796e-09,
      "elapsed_s": 11812.161114931107,
      "loss": 2.7574667930603027,
      "hidden": 0.17347335442900658,
      "layer": 0.0868199598044157,
      "mlm": 1.4555469751358032,
      "distill": 0.9115630835294724,
      "teacher_acc": 0.7215050607919693
    },
    {
      "step": 15911,
      "lr": 4.106139423612487e-09,
      "elapsed_s": 11819.616875171661,
      "loss": 2.794040083885193,
      "hidden": 0.16787945851683617,
      "layer": 0.08559676259756088,
      "mlm": 1.451412171125412,
      "distill": 0.964070588350296,
      "teacher_acc": 0.7366849184036255
    },
    {
      "step": 15921,
      "lr": 3.2443757156330744e-09,
      "elapsed_s": 11827.041406154633,
      "loss": 2.7593058943748474,
      "hidden": 0.17117313295602798,
      "layer": 0.08550054579973221,
      "mlm": 1.4391368627548218,
      "distill": 0.9350724518299103,
      "teacher_acc": 0.7349251955747604
    },
    {
      "step": 15931,
      "lr": 2.4839877497406792e-09,
      "elapsed_s": 11834.427202939987,
      "loss": 2.7896527647972107,
      "hidden": 0.17236778885126114,
      "layer": 0.08558817766606808,
      "mlm": 1.4373078346252441,
      "distill": 0.9648153185844421,
      "teacher_acc": 0.7295483499765396
    },
    {
      "step": 15941,
      "lr": 1.8249786097324084e-09,
      "elapsed_s": 11841.825204610825,
      "loss": 2.8704652786254883,
      "hidden": 0.17005695402622223,
      "layer": 0.08615738712251186,
      "mlm": 1.4916811287403107,
      "distill": 0.9955915659666061,
      "teacher_acc": 0.7347802966833115
    },
    {
      "step": 15951,
      "lr": 1.2673509682564755e-09,
      "elapsed_s": 11849.230217218399,
      "loss": 2.799567937850952,
      "hidden": 0.1709761656820774,
      "layer": 0.08514441922307014,
      "mlm": 1.4847007095813751,
      "distill": 0.9303427338600159,
      "teacher_acc": 0.7220852971076965
    },
    {
      "step": 15961,
      "lr": 8.111070868010995e-10,
      "elapsed_s": 11856.628919363022,
      "loss": 2.7967717051506042,
      "hidden": 0.1717565394937992,
      "layer": 0.08530361950397491,
      "mlm": 1.4237821400165558,
      "distill": 0.9868247359991074,
      "teacher_acc": 0.7447926998138428
    },
    {
      "step": 15971,
      "lr": 4.5624881569172885e-10,
      "elapsed_s": 11863.996557712555,
      "loss": 2.7791637778282166,
      "hidden": 0.16775130853056908,
      "layer": 0.08401280827820301,
      "mlm": 1.5047166645526886,
      "distill": 0.8969380706548691,
      "teacher_acc": 0.7125802636146545
    },
    {
      "step": 15981,
      "lr": 2.027775940743881e-10,
      "elapsed_s": 11871.401898145676,
      "loss": 2.748114824295044,
      "hidden": 0.1705903746187687,
      "layer": 0.08697794377803802,
      "mlm": 1.3905678391456604,
      "distill": 0.9728772938251495,
      "teacher_acc": 0.7344939261674881
    },
    {
      "step": 15991,
      "lr": 5.0694449918453625e-11,
      "elapsed_s": 11878.81341791153,
      "loss": 2.6740256547927856,
      "hidden": 0.1680052988231182,
      "layer": 0.0850144512951374,
      "mlm": 1.376772940158844,
      "distill": 0.9187349230051041,
      "teacher_acc": 0.744279220700264
    },
    {
      "step": 16000,
      "lr": 5.069446695760149e-13,
      "elapsed_s": 11885.497461795807,
      "loss": 2.7582311034202576,
      "hidden": 0.16607901453971863,
      "layer": 0.08408323675394058,
      "mlm": 1.4867496192455292,
      "distill": 0.8972818106412888,
      "teacher_acc": 0.7201758027076721
    }
  ]
}