{
  "source": {
    "issue": "https://github.com/marin-community/marin/issues/8435",
    "scaling_script_commit": "d23e6e9c3673435fb82d83aa6c51a607d0da6009",
    "loss_key": "eval_dropless/paloma/macro_loss",
    "bpb_key": "eval_dropless/paloma/macro_bpb",
    "runs": {
      "d768": {
        "display_name": "rav-ladder-d768-v2",
        "run_id": "rav-ladder-d768-v2",
        "url": "https://wandb.ai/marin-community/marin_moe/runs/rav-ladder-d768-v2",
        "stop_after_steps": 11420,
        "paired_observations": 20
      },
      "d1024": {
        "display_name": "rav-ladder-d1024",
        "run_id": "rav-ladder-d1024",
        "url": "https://wandb.ai/marin-community/marin_moe/runs/rav-ladder-d1024",
        "stop_after_steps": 15276,
        "paired_observations": 20
      },
      "d1536": {
        "display_name": "rav-ladder-d1536",
        "run_id": "rav-ladder-d1536",
        "url": "https://wandb.ai/marin-community/marin_moe/runs/rav-ladder-d1536",
        "stop_after_steps": 15128,
        "paired_observations": 21
      },
      "d2048": {
        "display_name": "rav-ladder-d2048-v3",
        "run_id": "rav-ladder-d2048-v3",
        "url": "https://wandb.ai/marin-community/marin_moe/runs/rav-ladder-d2048-v3",
        "stop_after_steps": 20072,
        "paired_observations": 16
      }
    }
  },
  "published_loss_method": {
    "asymptote": 1.5,
    "d2048_correction": 0.005,
    "rung_final_losses": {
      "d768": 3.015353202819824,
      "d1024": 2.780874729156494,
      "d1536": 2.5637738704681396,
      "d2048": 2.4087081241607677
    },
    "amplitude": 84.78759324113464,
    "alpha": 0.08997567660738807,
    "hero_compute": 2.6108941613772703e+24,
    "final_loss": 2.0387845986743045
  },
  "calibration_rows": [
    {
      "rung": "d768",
      "step": 570,
      "progress": 0.049912434325744305,
      "loss": 3.786311388015747,
      "bpb": 1.3525993824005127
    },
    {
      "rung": "d768",
      "step": 1141,
      "progress": 0.09991243432574431,
      "loss": 3.539341926574707,
      "bpb": 1.266406774520874
    },
    {
      "rung": "d768",
      "step": 1712,
      "progress": 0.1499124343257443,
      "loss": 3.4669861793518066,
      "bpb": 1.241546392440796
    },
    {
      "rung": "d768",
      "step": 2283,
      "progress": 0.19991243432574432,
      "loss": 3.4246766567230225,
      "bpb": 1.227675437927246
    },
    {
      "rung": "d768",
      "step": 2854,
      "progress": 0.2499124343257443,
      "loss": 3.3789491653442383,
      "bpb": 1.2107453346252441
    },
    {
      "rung": "d768",
      "step": 3425,
      "progress": 0.2999124343257443,
      "loss": 3.347630023956299,
      "bpb": 1.2002332210540771
    },
    {
      "rung": "d768",
      "step": 3996,
      "progress": 0.3499124343257443,
      "loss": 3.3215203285217285,
      "bpb": 1.1904218196868896
    },
    {
      "rung": "d768",
      "step": 4567,
      "progress": 0.3999124343257443,
      "loss": 3.3007681369781494,
      "bpb": 1.1838829517364502
    },
    {
      "rung": "d768",
      "step": 5138,
      "progress": 0.4499124343257443,
      "loss": 3.265657901763916,
      "bpb": 1.1706867218017578
    },
    {
      "rung": "d768",
      "step": 5709,
      "progress": 0.4999124343257443,
      "loss": 3.246354103088379,
      "bpb": 1.1645169258117676
    },
    {
      "rung": "d768",
      "step": 6280,
      "progress": 0.5499124343257443,
      "loss": 3.219543218612671,
      "bpb": 1.1548465490341187
    },
    {
      "rung": "d768",
      "step": 6851,
      "progress": 0.5999124343257443,
      "loss": 3.197269916534424,
      "bpb": 1.1469311714172363
    },
    {
      "rung": "d768",
      "step": 7422,
      "progress": 0.6499124343257443,
      "loss": 3.1782631874084473,
      "bpb": 1.140868067741394
    },
    {
      "rung": "d768",
      "step": 7993,
      "progress": 0.6999124343257443,
      "loss": 3.149127960205078,
      "bpb": 1.1303406953811646
    },
    {
      "rung": "d768",
      "step": 8564,
      "progress": 0.7499124343257443,
      "loss": 3.127243995666504,
      "bpb": 1.1227161884307861
    },
    {
      "rung": "d768",
      "step": 9135,
      "progress": 0.7999124343257443,
      "loss": 3.106234073638916,
      "bpb": 1.1155595779418945
    },
    {
      "rung": "d768",
      "step": 9706,
      "progress": 0.8499124343257443,
      "loss": 3.0818402767181396,
      "bpb": 1.107328176498413
    },
    {
      "rung": "d768",
      "step": 10277,
      "progress": 0.8999124343257443,
      "loss": 3.0587244033813477,
      "bpb": 1.0991122722625732
    },
    {
      "rung": "d768",
      "step": 10848,
      "progress": 0.9499124343257443,
      "loss": 3.038463592529297,
      "bpb": 1.0922293663024902
    },
    {
      "rung": "d768",
      "step": 11419,
      "progress": 0.9999124343257443,
      "loss": 3.015353202819824,
      "bpb": 1.0837204456329346
    },
    {
      "rung": "d1024",
      "step": 763,
      "progress": 0.04994763026970411,
      "loss": 3.4264705181121826,
      "bpb": 1.2265902757644653
    },
    {
      "rung": "d1024",
      "step": 1527,
      "progress": 0.09996072270227808,
      "loss": 3.264733076095581,
      "bpb": 1.171379804611206
    },
    {
      "rung": "d1024",
      "step": 2291,
      "progress": 0.14997381513485206,
      "loss": 3.18711519241333,
      "bpb": 1.1436543464660645
    },
    {
      "rung": "d1024",
      "step": 3055,
      "progress": 0.19998690756742601,
      "loss": 3.144479513168335,
      "bpb": 1.1281962394714355
    },
    {
      "rung": "d1024",
      "step": 3819,
      "progress": 0.25,
      "loss": 3.1228179931640625,
      "bpb": 1.1218509674072266
    },
    {
      "rung": "d1024",
      "step": 4583,
      "progress": 0.30001309243257396,
      "loss": 3.0838675498962402,
      "bpb": 1.1073096990585327
    },
    {
      "rung": "d1024",
      "step": 5347,
      "progress": 0.35002618486514797,
      "loss": 3.0643138885498047,
      "bpb": 1.1006712913513184
    },
    {
      "rung": "d1024",
      "step": 6111,
      "progress": 0.40003927729772193,
      "loss": 3.048776149749756,
      "bpb": 1.0964221954345703
    },
    {
      "rung": "d1024",
      "step": 6875,
      "progress": 0.4500523697302959,
      "loss": 3.0225400924682617,
      "bpb": 1.0868868827819824
    },
    {
      "rung": "d1024",
      "step": 7639,
      "progress": 0.5000654621628698,
      "loss": 2.993273973464966,
      "bpb": 1.0762288570404053
    },
    {
      "rung": "d1024",
      "step": 8403,
      "progress": 0.5500785545954439,
      "loss": 2.97526478767395,
      "bpb": 1.0697720050811768
    },
    {
      "rung": "d1024",
      "step": 9167,
      "progress": 0.6000916470280178,
      "loss": 2.9497318267822266,
      "bpb": 1.0606828927993774
    },
    {
      "rung": "d1024",
      "step": 9931,
      "progress": 0.6501047394605918,
      "loss": 2.9277210235595703,
      "bpb": 1.05281400680542
    },
    {
      "rung": "d1024",
      "step": 10695,
      "progress": 0.7001178318931658,
      "loss": 2.907698392868042,
      "bpb": 1.0459961891174316
    },
    {
      "rung": "d1024",
      "step": 11459,
      "progress": 0.7501309243257397,
      "loss": 2.8838517665863037,
      "bpb": 1.037614107131958
    },
    {
      "rung": "d1024",
      "step": 12223,
      "progress": 0.8001440167583137,
      "loss": 2.8643991947174072,
      "bpb": 1.031097412109375
    },
    {
      "rung": "d1024",
      "step": 12987,
      "progress": 0.8501571091908877,
      "loss": 2.8425564765930176,
      "bpb": 1.0236093997955322
    },
    {
      "rung": "d1024",
      "step": 13751,
      "progress": 0.9001702016234616,
      "loss": 2.8220372200012207,
      "bpb": 1.016756534576416
    },
    {
      "rung": "d1024",
      "step": 14515,
      "progress": 0.9501832940560356,
      "loss": 2.7989492416381836,
      "bpb": 1.0083966255187988
    },
    {
      "rung": "d1024",
      "step": 15275,
      "progress": 0.9999345378371302,
      "loss": 2.780874729156494,
      "bpb": 1.0022166967391968
    },
    {
      "rung": "d1536",
      "step": 755,
      "progress": 0.04990745637228979,
      "loss": 3.1583311557769775,
      "bpb": 1.1309518814086914
    },
    {
      "rung": "d1536",
      "step": 1511,
      "progress": 0.09988101533580117,
      "loss": 3.001892566680908,
      "bpb": 1.0771872997283936
    },
    {
      "rung": "d1536",
      "step": 2267,
      "progress": 0.14985457429931254,
      "loss": 2.944444179534912,
      "bpb": 1.0574387311935425
    },
    {
      "rung": "d1536",
      "step": 3023,
      "progress": 0.1998281332628239,
      "loss": 2.9032158851623535,
      "bpb": 1.0426092147827148
    },
    {
      "rung": "d1536",
      "step": 3779,
      "progress": 0.24980169222633528,
      "loss": 2.8789801597595215,
      "bpb": 1.0346770286560059
    },
    {
      "rung": "d1536",
      "step": 4535,
      "progress": 0.29977525118984666,
      "loss": 2.8418924808502197,
      "bpb": 1.0214356184005737
    },
    {
      "rung": "d1536",
      "step": 5291,
      "progress": 0.349748810153358,
      "loss": 2.8284969329833984,
      "bpb": 1.0172958374023438
    },
    {
      "rung": "d1536",
      "step": 6047,
      "progress": 0.39972236911686937,
      "loss": 2.800361156463623,
      "bpb": 1.0071771144866943
    },
    {
      "rung": "d1536",
      "step": 6803,
      "progress": 0.4496959280803807,
      "loss": 2.7863872051239014,
      "bpb": 1.0027999877929688
    },
    {
      "rung": "d1536",
      "step": 7559,
      "progress": 0.49966948704389214,
      "loss": 2.75604510307312,
      "bpb": 0.9912127256393433
    },
    {
      "rung": "d1536",
      "step": 8315,
      "progress": 0.5496430460074034,
      "loss": 2.7345495223999023,
      "bpb": 0.9838782548904419
    },
    {
      "rung": "d1536",
      "step": 9071,
      "progress": 0.5996166049709148,
      "loss": 2.714014768600464,
      "bpb": 0.9768331050872803
    },
    {
      "rung": "d1536",
      "step": 9827,
      "progress": 0.6495901639344263,
      "loss": 2.6984610557556152,
      "bpb": 0.9718459248542786
    },
    {
      "rung": "d1536",
      "step": 10583,
      "progress": 0.6995637228979376,
      "loss": 2.6755783557891846,
      "bpb": 0.9634991884231567
    },
    {
      "rung": "d1536",
      "step": 11339,
      "progress": 0.749537281861449,
      "loss": 2.6567752361297607,
      "bpb": 0.9570150971412659
    },
    {
      "rung": "d1536",
      "step": 12095,
      "progress": 0.7995108408249604,
      "loss": 2.638200283050537,
      "bpb": 0.9506324529647827
    },
    {
      "rung": "d1536",
      "step": 12851,
      "progress": 0.8494843997884717,
      "loss": 2.619727373123169,
      "bpb": 0.9444899559020996
    },
    {
      "rung": "d1536",
      "step": 13607,
      "progress": 0.8994579587519831,
      "loss": 2.599482536315918,
      "bpb": 0.9375103712081909
    },
    {
      "rung": "d1536",
      "step": 14363,
      "progress": 0.9494315177154945,
      "loss": 2.579882860183716,
      "bpb": 0.9306644201278687
    },
    {
      "rung": "d1536",
      "step": 15119,
      "progress": 0.9994050766790058,
      "loss": 2.5638508796691895,
      "bpb": 0.9250718355178833
    },
    {
      "rung": "d1536",
      "step": 15127,
      "progress": 0.9999338974087785,
      "loss": 2.5637738704681396,
      "bpb": 0.9250680208206177
    },
    {
      "rung": "d2048",
      "step": 1003,
      "progress": 0.04997010761259466,
      "loss": 2.9312033653259277,
      "bpb": 1.052567958831787
    },
    {
      "rung": "d2048",
      "step": 2007,
      "progress": 0.09999003587086489,
      "loss": 2.802788257598877,
      "bpb": 1.0079604387283325
    },
    {
      "rung": "d2048",
      "step": 3011,
      "progress": 0.15000996412913511,
      "loss": 2.7470338344573975,
      "bpb": 0.9881319999694824
    },
    {
      "rung": "d2048",
      "step": 4015,
      "progress": 0.20002989238740534,
      "loss": 2.710448741912842,
      "bpb": 0.9754750728607178
    },
    {
      "rung": "d2048",
      "step": 5019,
      "progress": 0.25004982064567555,
      "loss": 2.6807942390441895,
      "bpb": 0.9656500816345215
    },
    {
      "rung": "d2048",
      "step": 6023,
      "progress": 0.3000697489039458,
      "loss": 2.6612348556518555,
      "bpb": 0.958776593208313
    },
    {
      "rung": "d2048",
      "step": 7027,
      "progress": 0.350089677162216,
      "loss": 2.643836498260498,
      "bpb": 0.952675461769104
    },
    {
      "rung": "d2048",
      "step": 8031,
      "progress": 0.40010960542048624,
      "loss": 2.6150641441345215,
      "bpb": 0.9423530697822571
    },
    {
      "rung": "d2048",
      "step": 9035,
      "progress": 0.45012953367875647,
      "loss": 2.6094987392425537,
      "bpb": 0.9413461685180664
    },
    {
      "rung": "d2048",
      "step": 10039,
      "progress": 0.5001494619370267,
      "loss": 2.590702533721924,
      "bpb": 0.934929609298706
    },
    {
      "rung": "d2048",
      "step": 11043,
      "progress": 0.550169390195297,
      "loss": 2.5618793964385986,
      "bpb": 0.9237306714057922
    },
    {
      "rung": "d2048",
      "step": 12047,
      "progress": 0.6001893184535672,
      "loss": 2.543623447418213,
      "bpb": 0.9173685908317566
    },
    {
      "rung": "d2048",
      "step": 13051,
      "progress": 0.6502092467118374,
      "loss": 2.527618408203125,
      "bpb": 0.9121881127357483
    },
    {
      "rung": "d2048",
      "step": 14055,
      "progress": 0.7002291749701076,
      "loss": 2.5085854530334473,
      "bpb": 0.9054526686668396
    },
    {
      "rung": "d2048",
      "step": 15059,
      "progress": 0.7502491032283778,
      "loss": 2.4904448986053467,
      "bpb": 0.8991650342941284
    },
    {
      "rung": "d2048",
      "step": 16063,
      "progress": 0.8002690314866481,
      "loss": 2.474504232406616,
      "bpb": 0.8939169645309448
    }
  ],
  "calibration_models": {
    "all_observations": {
      "intercept": 0.02534206814355754,
      "slope": 0.3508353365580562,
      "r_squared": 0.9999492787158349,
      "rmse": 0.0007156457563692617,
      "max_abs_residual": 0.0024443607336765982,
      "n": 77
    },
    "progress_ge_60_percent": {
      "intercept": 0.023111985828162478,
      "slope": 0.3517620470280023,
      "r_squared": 0.9999710230801817,
      "rmse": 0.00041804574296059243,
      "max_abs_residual": 0.0009589594514112232,
      "n": 31
    },
    "last_observation_per_rung": {
      "intercept": 0.024403703813305484,
      "slope": 0.3514057124730566,
      "r_squared": 0.9999762175717344,
      "rmse": 0.00035855099890475873,
      "max_abs_residual": 0.000597727428335082,
      "n": 4
    }
  },
  "forecast_bpb": {
    "primary": 0.7406197489888388,
    "sensitivity_models": {
      "all_observations": 0.7406197489888388,
      "progress_ge_60_percent": 0.7402790297069999,
      "last_observation_per_rung": 0.7408442582895441
    },
    "cluster_bootstrap_95_percent": [
      0.7404078432532418,
      0.7431608412460924
    ],
    "leave_one_rung_out": {
      "d768": {
        "model": {
          "intercept": 0.02548018628229475,
          "slope": 0.350764767186531,
          "r_squared": 0.9998943663429595,
          "rmse": 0.0007673091652648126,
          "max_abs_residual": 0.002359597427682214,
          "n": 57
        },
        "forecast_bpb": 0.7406139913797722
      },
      "d1024": {
        "model": {
          "intercept": 0.02515205675428795,
          "slope": 0.350827800319186,
          "r_squared": 0.9999664310980114,
          "rmse": 0.0006383185469817722,
          "max_abs_residual": 0.002230547406385952,
          "n": 57
        },
        "forecast_bpb": 0.7404143728318286
      },
      "d1536": {
        "model": {
          "intercept": 0.026076895722187694,
          "slope": 0.3506601452952177,
          "r_squared": 0.9999649769316483,
          "rmse": 0.0006205303462192168,
          "max_abs_residual": 0.0013651348654215578,
          "n": 56
        },
        "forecast_bpb": 0.7409973993189715
      },
      "d2048": {
        "model": {
          "intercept": 0.025865103787149128,
          "slope": 0.3506814291484095,
          "r_squared": 0.999935574264911,
          "rmse": 0.0007488746416819364,
          "max_abs_residual": 0.0024813058102761687,
          "n": 61
        },
        "forecast_bpb": 0.7408290005760207
      }
    }
  },
  "current_hero_check": {
    "step": 45837,
    "loss": 2.25874662399292,
    "bpb": 0.8187980651855469,
    "predicted_bpb_from_all_observations": 0.8177902001714868
  }
}