{ "method": { "description": "Fit each subset BPB as a linear function of dropless Paloma macro-loss across all paired ladder evaluations, then compose that calibration with the preregistered final macro-loss scaling law.", "macro_scaling_law": { "asymptote": 1.5, "amplitude": 84.78759324113464, "alpha": 0.08997567660738807, "hero_compute": 2.6108941613772703e+24, "hero_final_macro_loss": 2.0387845986743045 }, "open_weight_compute": { "method": "6 \u00d7 active parameters \u00d7 training tokens", "qwen": { "active_parameters": 3000000000, "training_tokens": 36000000000000, "estimated_training_flops": 6.48e+23, "caveat": "The 36T-token count is an input assumption." }, "nemotron_nano": { "active_parameters": 3500000000, "training_tokens": 25000000000000, "estimated_training_flops": 5.25e+23, "caveat": "NVIDIA publishes 25T training tokens and 3.5B active parameters, not an exact training-FLOP total. The estimate uses the standard 6ND approximation; exact hybrid-MoE compute may differ." }, "nemotron_super": { "active_parameters": 12000000000, "training_tokens": 25000000000000, "estimated_training_flops": 1.8e+24, "caveat": "NVIDIA reports over 25T training tokens and 12B active parameters. The 1.8e24-FLOP coordinate uses 25T, so it is a lower-bound 6ND estimate; exact hybrid-MoE compute may differ." } } }, "runs": { "d768": { "display_name": "rav-ladder-d768-v2", "run_id": "rav-ladder-d768-v2", "url": "https://wandb.ai/marin-community/marin_moe/runs/rav-ladder-d768-v2", "stop_after_steps": 11420, "paired_observations": 20, "full_compute": 2.649679880270119e+19 }, "d1024": { "display_name": "rav-ladder-d1024", "run_id": "rav-ladder-d1024", "url": "https://wandb.ai/marin-community/marin_moe/runs/rav-ladder-d1024", "stop_after_steps": 15276, "paired_observations": 20, "full_compute": 1.7261182741798453e+20 }, "d1536": { "display_name": "rav-ladder-d1536", "run_id": "rav-ladder-d1536", "url": "https://wandb.ai/marin-community/marin_moe/runs/rav-ladder-d1536", "stop_after_steps": 15128, "paired_observations": 21, "full_compute": 1.3813258500033133e+21 }, "d2048": { "display_name": "rav-ladder-d2048-v3", "run_id": "rav-ladder-d2048-v3", "url": "https://wandb.ai/marin-community/marin_moe/runs/rav-ladder-d2048-v3", "stop_after_steps": 20072, "paired_observations": 16, "full_compute": 7.741814947656449e+21 } }, "calibration_rows": [ { "rung": "d768", "step": 570, "progress": 0.049912434325744305, "compute": 1.3225197300822835e+18, "macro_loss": 3.786311388015747, "subset_bpb": { "4chan": 1.3381925821304321, "C4 100 domains": 1.183597207069397, "C4 English": 1.1980968713760376, "Dolma v1.5": 1.15755033493042, "100 programming languages": 0.8471120595932007, "100 subreddits": 1.358364462852478, "Falcon RefinedWeb": 1.269904375076294, "Gab": 2.0462453365325928, "M2D2 S2ORC": 1.088677167892456, "M2D2 Wikipedia": 1.161108136177063, "Manosphere": 1.4024579524993896, "mC4": 1.186732530593872, "Penn Treebank": 1.362096905708313, "RedPajama": 1.0889370441436768, "Twitter AAE": 2.7646725177764893, "WikiText-103": 1.1878461837768555 } }, { "rung": "d768", "step": 1141, "progress": 0.09991243432574431, "compute": 2.647359670217343e+18, "macro_loss": 3.539341926574707, "subset_bpb": { "4chan": 1.265560269355774, "C4 100 domains": 1.0967395305633545, "C4 English": 1.1192233562469482, "Dolma v1.5": 1.0739890336990356, "100 programming languages": 0.7561141848564148, "100 subreddits": 1.281484842300415, "Falcon RefinedWeb": 1.187713861465454, "Gab": 1.938223958015442, "M2D2 S2ORC": 1.0147650241851807, "M2D2 Wikipedia": 1.0707225799560547, "Manosphere": 1.3255894184112549, "mC4": 1.1015477180480957, "Penn Treebank": 1.268905520439148, "RedPajama": 0.9981678128242493, "Twitter AAE": 2.6762170791625977, "WikiText-103": 1.0875455141067505 } }, { "rung": "d768", "step": 1712, "progress": 0.1499124343257443, "compute": 3.9721996103524024e+18, "macro_loss": 3.4669861793518066, "subset_bpb": { "4chan": 1.2435483932495117, "C4 100 domains": 1.0691988468170166, "C4 English": 1.0932269096374512, "Dolma v1.5": 1.0489007234573364, "100 programming languages": 0.7334582209587097, "100 subreddits": 1.2645055055618286, "Falcon RefinedWeb": 1.1609680652618408, "Gab": 1.9038922786712646, "M2D2 S2ORC": 0.9888909459114075, "M2D2 Wikipedia": 1.042048692703247, "Manosphere": 1.3051522970199585, "mC4": 1.0760904550552368, "Penn Treebank": 1.2433375120162964, "RedPajama": 0.9720095992088318, "Twitter AAE": 2.6675612926483154, "WikiText-103": 1.0519531965255737 } }, { "rung": "d768", "step": 2283, "progress": 0.19991243432574432, "compute": 5.297039550487462e+18, "macro_loss": 3.4246766567230225, "subset_bpb": { "4chan": 1.2292804718017578, "C4 100 domains": 1.0553934574127197, "C4 English": 1.080311894416809, "Dolma v1.5": 1.0324463844299316, "100 programming languages": 0.7146015167236328, "100 subreddits": 1.250195860862732, "Falcon RefinedWeb": 1.1475178003311157, "Gab": 1.8935010433197021, "M2D2 S2ORC": 0.973348081111908, "M2D2 Wikipedia": 1.0238665342330933, "Manosphere": 1.2933075428009033, "mC4": 1.0603535175323486, "Penn Treebank": 1.2074311971664429, "RedPajama": 0.9542980194091797, "Twitter AAE": 2.692934036254883, "WikiText-103": 1.0340183973312378 } }, { "rung": "d768", "step": 2854, "progress": 0.2499124343257443, "compute": 6.621879490622521e+18, "macro_loss": 3.3789491653442383, "subset_bpb": { "4chan": 1.2149229049682617, "C4 100 domains": 1.0434919595718384, "C4 English": 1.0679527521133423, "Dolma v1.5": 1.0202425718307495, "100 programming languages": 0.7009655833244324, "100 subreddits": 1.2408450841903687, "Falcon RefinedWeb": 1.1341123580932617, "Gab": 1.8647844791412354, "M2D2 S2ORC": 0.961579442024231, "M2D2 Wikipedia": 1.0103931427001953, "Manosphere": 1.2795796394348145, "mC4": 1.0443986654281616, "Penn Treebank": 1.1924655437469482, "RedPajama": 0.9407017827033997, "Twitter AAE": 2.635784149169922, "WikiText-103": 1.0197054147720337 } }, { "rung": "d768", "step": 3425, "progress": 0.2999124343257443, "compute": 7.946719430757581e+18, "macro_loss": 3.347630023956299, "subset_bpb": { "4chan": 1.2003980875015259, "C4 100 domains": 1.0301345586776733, "C4 English": 1.0559265613555908, "Dolma v1.5": 1.0090357065200806, "100 programming languages": 0.6889880299568176, "100 subreddits": 1.227866768836975, "Falcon RefinedWeb": 1.1219191551208496, "Gab": 1.8524492979049683, "M2D2 S2ORC": 0.9507759213447571, "M2D2 Wikipedia": 0.9988580346107483, "Manosphere": 1.2693324089050293, "mC4": 1.033578634262085, "Penn Treebank": 1.1817010641098022, "RedPajama": 0.9285488724708557, "Twitter AAE": 2.6466569900512695, "WikiText-103": 1.0075613260269165 } }, { "rung": "d768", "step": 3996, "progress": 0.3499124343257443, "compute": 9.27155937089264e+18, "macro_loss": 3.3215203285217285, "subset_bpb": { "4chan": 1.1908609867095947, "C4 100 domains": 1.0230196714401245, "C4 English": 1.0496577024459839, "Dolma v1.5": 0.9996422529220581, "100 programming languages": 0.6834163069725037, "100 subreddits": 1.2198172807693481, "Falcon RefinedWeb": 1.114116907119751, "Gab": 1.83376944065094, "M2D2 S2ORC": 0.9441012740135193, "M2D2 Wikipedia": 0.9908115267753601, "Manosphere": 1.2597565650939941, "mC4": 1.025875449180603, "Penn Treebank": 1.1832106113433838, "RedPajama": 0.9195191860198975, "Twitter AAE": 2.610264539718628, "WikiText-103": 0.998910665512085 } }, { "rung": "d768", "step": 4567, "progress": 0.3999124343257443, "compute": 1.05963993110277e+19, "macro_loss": 3.3007681369781494, "subset_bpb": { "4chan": 1.1925668716430664, "C4 100 domains": 1.0141232013702393, "C4 English": 1.0419036149978638, "Dolma v1.5": 0.9923625588417053, "100 programming languages": 0.6724739074707031, "100 subreddits": 1.2141919136047363, "Falcon RefinedWeb": 1.1063867807388306, "Gab": 1.8252196311950684, "M2D2 S2ORC": 0.9361984133720398, "M2D2 Wikipedia": 0.9820300936698914, "Manosphere": 1.2536715269088745, "mC4": 1.0177823305130005, "Penn Treebank": 1.1631258726119995, "RedPajama": 0.912035346031189, "Twitter AAE": 2.6306567192077637, "WikiText-103": 0.9873964190483093 } }, { "rung": "d768", "step": 5138, "progress": 0.4499124343257443, "compute": 1.192123925116276e+19, "macro_loss": 3.265657901763916, "subset_bpb": { "4chan": 1.1766650676727295, "C4 100 domains": 1.0054348707199097, "C4 English": 1.0332576036453247, "Dolma v1.5": 0.9824267029762268, "100 programming languages": 0.6632277965545654, "100 subreddits": 1.2048587799072266, "Falcon RefinedWeb": 1.0974633693695068, "Gab": 1.8070851564407349, "M2D2 S2ORC": 0.926234245300293, "M2D2 Wikipedia": 0.9713019132614136, "Manosphere": 1.2431429624557495, "mC4": 1.0083078145980835, "Penn Treebank": 1.155056118965149, "RedPajama": 0.9016720056533813, "Twitter AAE": 2.578742504119873, "WikiText-103": 0.9761102795600891 } }, { "rung": "d768", "step": 5709, "progress": 0.4999124343257443, "compute": 1.3246079191297819e+19, "macro_loss": 3.246354103088379, "subset_bpb": { "4chan": 1.172190546989441, "C4 100 domains": 0.9969291090965271, "C4 English": 1.02510404586792, "Dolma v1.5": 0.9742732048034668, "100 programming languages": 0.6537235975265503, "100 subreddits": 1.1980420351028442, "Falcon RefinedWeb": 1.0887842178344727, "Gab": 1.801762580871582, "M2D2 S2ORC": 0.918914258480072, "M2D2 Wikipedia": 0.9624807238578796, "Manosphere": 1.2359827756881714, "mC4": 0.998288631439209, "Penn Treebank": 1.146346092224121, "RedPajama": 0.8922416567802429, "Twitter AAE": 2.5981993675231934, "WikiText-103": 0.9690070152282715 } }, { "rung": "d768", "step": 6280, "progress": 0.5499124343257443, "compute": 1.4570919131432878e+19, "macro_loss": 3.219543218612671, "subset_bpb": { "4chan": 1.1652041673660278, "C4 100 domains": 0.9894940257072449, "C4 English": 1.0182206630706787, "Dolma v1.5": 0.9665247201919556, "100 programming languages": 0.645328938961029, "100 subreddits": 1.189145803451538, "Falcon RefinedWeb": 1.081608533859253, "Gab": 1.7908577919006348, "M2D2 S2ORC": 0.9129423499107361, "M2D2 Wikipedia": 0.9539775848388672, "Manosphere": 1.226708173751831, "mC4": 0.9913347363471985, "Penn Treebank": 1.1339954137802124, "RedPajama": 0.8844280242919922, "Twitter AAE": 2.5709121227264404, "WikiText-103": 0.9568619132041931 } }, { "rung": "d768", "step": 6851, "progress": 0.5999124343257443, "compute": 1.5895759071567938e+19, "macro_loss": 3.197269916534424, "subset_bpb": { "4chan": 1.1585569381713867, "C4 100 domains": 0.9816451668739319, "C4 English": 1.0110512971878052, "Dolma v1.5": 0.9586087465286255, "100 programming languages": 0.6380024552345276, "100 subreddits": 1.1830933094024658, "Falcon RefinedWeb": 1.0733813047409058, "Gab": 1.77723228931427, "M2D2 S2ORC": 0.9052955508232117, "M2D2 Wikipedia": 0.9470551013946533, "Manosphere": 1.2231262922286987, "mC4": 0.9844971299171448, "Penn Treebank": 1.1251364946365356, "RedPajama": 0.8770955801010132, "Twitter AAE": 2.558684825897217, "WikiText-103": 0.948436975479126 } }, { "rung": "d768", "step": 7422, "progress": 0.6499124343257443, "compute": 1.7220599011702997e+19, "macro_loss": 3.1782631874084473, "subset_bpb": { "4chan": 1.147130012512207, "C4 100 domains": 0.9736474752426147, "C4 English": 1.003769874572754, "Dolma v1.5": 0.9505559206008911, "100 programming languages": 0.630558431148529, "100 subreddits": 1.1766067743301392, "Falcon RefinedWeb": 1.0658626556396484, "Gab": 1.7767391204833984, "M2D2 S2ORC": 0.8969746828079224, "M2D2 Wikipedia": 0.9381488561630249, "Manosphere": 1.2161033153533936, "mC4": 0.9782578945159912, "Penn Treebank": 1.1148589849472046, "RedPajama": 0.8682696223258972, "Twitter AAE": 2.5774762630462646, "WikiText-103": 0.938930332660675 } }, { "rung": "d768", "step": 7993, "progress": 0.6999124343257443, "compute": 1.8545438951838056e+19, "macro_loss": 3.149127960205078, "subset_bpb": { "4chan": 1.1418535709381104, "C4 100 domains": 0.9650135636329651, "C4 English": 0.995804488658905, "Dolma v1.5": 0.9420391917228699, "100 programming languages": 0.6232466101646423, "100 subreddits": 1.1681023836135864, "Falcon RefinedWeb": 1.0573793649673462, "Gab": 1.7588070631027222, "M2D2 S2ORC": 0.8889942169189453, "M2D2 Wikipedia": 0.9288572072982788, "Manosphere": 1.2055999040603638, "mC4": 0.968883752822876, "Penn Treebank": 1.1028709411621094, "RedPajama": 0.859542965888977, "Twitter AAE": 2.54948091506958, "WikiText-103": 0.9289748668670654 } }, { "rung": "d768", "step": 8564, "progress": 0.7499124343257443, "compute": 1.9870278891973116e+19, "macro_loss": 3.127243995666504, "subset_bpb": { "4chan": 1.1341228485107422, "C4 100 domains": 0.9575449228286743, "C4 English": 0.9890409708023071, "Dolma v1.5": 0.9348679780960083, "100 programming languages": 0.6145836710929871, "100 subreddits": 1.1616744995117188, "Falcon RefinedWeb": 1.0497078895568848, "Gab": 1.7488986253738403, "M2D2 S2ORC": 0.8830623626708984, "M2D2 Wikipedia": 0.9215503334999084, "Manosphere": 1.1990224123001099, "mC4": 0.9618485569953918, "Penn Treebank": 1.0913602113723755, "RedPajama": 0.8524506688117981, "Twitter AAE": 2.5434796810150146, "WikiText-103": 0.9202432036399841 } }, { "rung": "d768", "step": 9135, "progress": 0.7999124343257443, "compute": 2.1195118832108175e+19, "macro_loss": 3.106234073638916, "subset_bpb": { "4chan": 1.1256409883499146, "C4 100 domains": 0.9497495293617249, "C4 English": 0.9814943075180054, "Dolma v1.5": 0.9269346594810486, "100 programming languages": 0.6062076687812805, "100 subreddits": 1.1540273427963257, "Falcon RefinedWeb": 1.0417910814285278, "Gab": 1.7419289350509644, "M2D2 S2ORC": 0.876534640789032, "M2D2 Wikipedia": 0.9130876064300537, "Manosphere": 1.1921172142028809, "mC4": 0.9542795419692993, "Penn Treebank": 1.0843861103057861, "RedPajama": 0.8435538411140442, "Twitter AAE": 2.5458500385284424, "WikiText-103": 0.9113688468933105 } }, { "rung": "d768", "step": 9706, "progress": 0.8499124343257443, "compute": 2.2519958772243235e+19, "macro_loss": 3.0818402767181396, "subset_bpb": { "4chan": 1.1187108755111694, "C4 100 domains": 0.9412102699279785, "C4 English": 0.9706609845161438, "Dolma v1.5": 0.9180509448051453, "100 programming languages": 0.5990607738494873, "100 subreddits": 1.1442501544952393, "Falcon RefinedWeb": 1.0314582586288452, "Gab": 1.7329875230789185, "M2D2 S2ORC": 0.8718366026878357, "M2D2 Wikipedia": 0.9033923745155334, "Manosphere": 1.1840054988861084, "mC4": 0.9428373575210571, "Penn Treebank": 1.0758256912231445, "RedPajama": 0.8363130688667297, "Twitter AAE": 2.5474278926849365, "WikiText-103": 0.8992229104042053 } }, { "rung": "d768", "step": 10277, "progress": 0.8999124343257443, "compute": 2.3844798712378294e+19, "macro_loss": 3.0587244033813477, "subset_bpb": { "4chan": 1.1095545291900635, "C4 100 domains": 0.9331361055374146, "C4 English": 0.9628610014915466, "Dolma v1.5": 0.9102556109428406, "100 programming languages": 0.5918354392051697, "100 subreddits": 1.137194275856018, "Falcon RefinedWeb": 1.0233373641967773, "Gab": 1.7282240390777588, "M2D2 S2ORC": 0.8651980757713318, "M2D2 Wikipedia": 0.8947835564613342, "Manosphere": 1.1772878170013428, "mC4": 0.9350122809410095, "Penn Treebank": 1.0682345628738403, "RedPajama": 0.8276615738868713, "Twitter AAE": 2.5304250717163086, "WikiText-103": 0.8907949924468994 } }, { "rung": "d768", "step": 10848, "progress": 0.9499124343257443, "compute": 2.5169638652513354e+19, "macro_loss": 3.038463592529297, "subset_bpb": { "4chan": 1.105111837387085, "C4 100 domains": 0.9255127310752869, "C4 English": 0.9555596709251404, "Dolma v1.5": 0.9029142260551453, "100 programming languages": 0.5839618444442749, "100 subreddits": 1.1308648586273193, "Falcon RefinedWeb": 1.0157238245010376, "Gab": 1.7158839702606201, "M2D2 S2ORC": 0.85860675573349, "M2D2 Wikipedia": 0.8868392109870911, "Manosphere": 1.169630765914917, "mC4": 0.927401065826416, "Penn Treebank": 1.0608580112457275, "RedPajama": 0.8201016783714294, "Twitter AAE": 2.5335114002227783, "WikiText-103": 0.8831877112388611 } }, { "rung": "d768", "step": 11419, "progress": 0.9999124343257443, "compute": 2.6494478592648413e+19, "macro_loss": 3.015353202819824, "subset_bpb": { "4chan": 1.1000752449035645, "C4 100 domains": 0.9186555743217468, "C4 English": 0.9491547346115112, "Dolma v1.5": 0.8959868550300598, "100 programming languages": 0.5766160488128662, "100 subreddits": 1.1243383884429932, "Falcon RefinedWeb": 1.0089563131332397, "Gab": 1.7025781869888306, "M2D2 S2ORC": 0.8529973030090332, "M2D2 Wikipedia": 0.8797162771224976, "Manosphere": 1.1631572246551514, "mC4": 0.9205182194709778, "Penn Treebank": 1.0533778667449951, "RedPajama": 0.8125020861625671, "Twitter AAE": 2.505134105682373, "WikiText-103": 0.8757628202438354 } }, { "rung": "d1024", "step": 763, "progress": 0.04994763026970411, "compute": 8.621551736051466e+18, "macro_loss": 3.4264705181121826, "subset_bpb": { "4chan": 1.229798436164856, "C4 100 domains": 1.061447262763977, "C4 English": 1.0863265991210938, "Dolma v1.5": 1.039535403251648, "100 programming languages": 0.7124236226081848, "100 subreddits": 1.2493720054626465, "Falcon RefinedWeb": 1.1516213417053223, "Gab": 1.8784056901931763, "M2D2 S2ORC": 0.9768689870834351, "M2D2 Wikipedia": 1.0312472581863403, "Manosphere": 1.290565848350525, "mC4": 1.065386414527893, "Penn Treebank": 1.2202571630477905, "RedPajama": 0.9610385298728943, "Twitter AAE": 2.632434368133545, "WikiText-103": 1.038715124130249 } }, { "rung": "d1024", "step": 1527, "progress": 0.09996072270227808, "compute": 1.7254403015662633e+19, "macro_loss": 3.264733076095581, "subset_bpb": { "4chan": 1.1809967756271362, "C4 100 domains": 1.0011171102523804, "C4 English": 1.0291398763656616, "Dolma v1.5": 0.9807115197181702, "100 programming languages": 0.6595858931541443, "100 subreddits": 1.2009297609329224, "Falcon RefinedWeb": 1.0941197872161865, "Gab": 1.8222253322601318, "M2D2 S2ORC": 0.9228313565254211, "M2D2 Wikipedia": 0.9676539897918701, "Manosphere": 1.2415175437927246, "mC4": 1.00575590133667, "Penn Treebank": 1.1531058549880981, "RedPajama": 0.8989117741584778, "Twitter AAE": 2.6174442768096924, "WikiText-103": 0.9660282731056213 } }, { "rung": "d1024", "step": 2291, "progress": 0.14997381513485206, "compute": 2.58872542952738e+19, "macro_loss": 3.18711519241333, "subset_bpb": { "4chan": 1.1574974060058594, "C4 100 domains": 0.9779878854751587, "C4 English": 1.0081391334533691, "Dolma v1.5": 0.9571076035499573, "100 programming languages": 0.6355056762695312, "100 subreddits": 1.1783812046051025, "Falcon RefinedWeb": 1.0702755451202393, "Gab": 1.7800179719924927, "M2D2 S2ORC": 0.9007173776626587, "M2D2 Wikipedia": 0.9424763321876526, "Manosphere": 1.2187564373016357, "mC4": 0.9819266200065613, "Penn Treebank": 1.1093310117721558, "RedPajama": 0.8753156661987305, "Twitter AAE": 2.562655448913574, "WikiText-103": 0.9423781633377075 } }, { "rung": "d1024", "step": 3055, "progress": 0.19998690756742601, "compute": 3.4520105574884966e+19, "macro_loss": 3.144479513168335, "subset_bpb": { "4chan": 1.1417996883392334, "C4 100 domains": 0.9653404355049133, "C4 English": 0.9954469203948975, "Dolma v1.5": 0.944088876247406, "100 programming languages": 0.6210877895355225, "100 subreddits": 1.1672292947769165, "Falcon RefinedWeb": 1.0565544366836548, "Gab": 1.753786325454712, "M2D2 S2ORC": 0.8886933326721191, "M2D2 Wikipedia": 0.9287380576133728, "Manosphere": 1.2043225765228271, "mC4": 0.9671258926391602, "Penn Treebank": 1.1028095483779907, "RedPajama": 0.861138641834259, "Twitter AAE": 2.5286505222320557, "WikiText-103": 0.9243259429931641 } }, { "rung": "d1024", "step": 3819, "progress": 0.25, "compute": 4.315295685449613e+19, "macro_loss": 3.1228179931640625, "subset_bpb": { "4chan": 1.1321505308151245, "C4 100 domains": 0.954188883304596, "C4 English": 0.985024094581604, "Dolma v1.5": 0.9343482851982117, "100 programming languages": 0.6137720942497253, "100 subreddits": 1.1576637029647827, "Falcon RefinedWeb": 1.0457321405410767, "Gab": 1.758316993713379, "M2D2 S2ORC": 0.8796892762184143, "M2D2 Wikipedia": 0.9178847074508667, "Manosphere": 1.1959971189498901, "mC4": 0.958915114402771, "Penn Treebank": 1.0840212106704712, "RedPajama": 0.8503111004829407, "Twitter AAE": 2.5687010288238525, "WikiText-103": 0.9128994941711426 } }, { "rung": "d1024", "step": 4583, "progress": 0.30001309243257396, "compute": 5.17858081341073e+19, "macro_loss": 3.0838675498962402, "subset_bpb": { "4chan": 1.1198930740356445, "C4 100 domains": 0.9443735480308533, "C4 English": 0.9761499166488647, "Dolma v1.5": 0.9245898723602295, "100 programming languages": 0.6044610738754272, "100 subreddits": 1.1480417251586914, "Falcon RefinedWeb": 1.0363966226577759, "Gab": 1.7340773344039917, "M2D2 S2ORC": 0.8695225119590759, "M2D2 Wikipedia": 0.9078037738800049, "Manosphere": 1.186063289642334, "mC4": 0.9472575187683105, "Penn Treebank": 1.0656424760818481, "RedPajama": 0.84007328748703, "Twitter AAE": 2.51196026802063, "WikiText-103": 0.900648832321167 } }, { "rung": "d1024", "step": 5347, "progress": 0.35002618486514797, "compute": 6.041865941371847e+19, "macro_loss": 3.0643138885498047, "subset_bpb": { "4chan": 1.115159511566162, "C4 100 domains": 0.9370368719100952, "C4 English": 0.9699608683586121, "Dolma v1.5": 0.91600102186203, "100 programming languages": 0.5966475605964661, "100 subreddits": 1.1408119201660156, "Falcon RefinedWeb": 1.0290876626968384, "Gab": 1.7253059148788452, "M2D2 S2ORC": 0.8623524308204651, "M2D2 Wikipedia": 0.9003553986549377, "Manosphere": 1.1785461902618408, "mC4": 0.9419435262680054, "Penn Treebank": 1.063533902168274, "RedPajama": 0.8316925168037415, "Twitter AAE": 2.5146403312683105, "WikiText-103": 0.8876654505729675 } }, { "rung": "d1024", "step": 6111, "progress": 0.40003927729772193, "compute": 6.905151069332963e+19, "macro_loss": 3.048776149749756, "subset_bpb": { "4chan": 1.106742024421692, "C4 100 domains": 0.9293282628059387, "C4 English": 0.9622703790664673, "Dolma v1.5": 0.9080584645271301, "100 programming languages": 0.5887786149978638, "100 subreddits": 1.1382540464401245, "Falcon RefinedWeb": 1.0218219757080078, "Gab": 1.7307031154632568, "M2D2 S2ORC": 0.8541252017021179, "M2D2 Wikipedia": 0.8899714946746826, "Manosphere": 1.1732336282730103, "mC4": 0.9333260655403137, "Penn Treebank": 1.0462347269058228, "RedPajama": 0.8227195739746094, "Twitter AAE": 2.5598888397216797, "WikiText-103": 0.8772980570793152 } }, { "rung": "d1024", "step": 6875, "progress": 0.4500523697302959, "compute": 7.76843619729408e+19, "macro_loss": 3.0225400924682617, "subset_bpb": { "4chan": 1.0996980667114258, "C4 100 domains": 0.9207890033721924, "C4 English": 0.9545198082923889, "Dolma v1.5": 0.8998680710792542, "100 programming languages": 0.579544186592102, "100 subreddits": 1.1286215782165527, "Falcon RefinedWeb": 1.0130637884140015, "Gab": 1.720096468925476, "M2D2 S2ORC": 0.8469160795211792, "M2D2 Wikipedia": 0.881537914276123, "Manosphere": 1.1647738218307495, "mC4": 0.9248369336128235, "Penn Treebank": 1.039719581604004, "RedPajama": 0.8152631521224976, "Twitter AAE": 2.53234601020813, "WikiText-103": 0.8685949444770813 } }, { "rung": "d1024", "step": 7639, "progress": 0.5000654621628698, "compute": 8.631721325255197e+19, "macro_loss": 2.993273973464966, "subset_bpb": { "4chan": 1.0887842178344727, "C4 100 domains": 0.9131224155426025, "C4 English": 0.9471720457077026, "Dolma v1.5": 0.8919450044631958, "100 programming languages": 0.5725046396255493, "100 subreddits": 1.11937415599823, "Falcon RefinedWeb": 1.0049573183059692, "Gab": 1.6974025964736938, "M2D2 S2ORC": 0.8408389687538147, "M2D2 Wikipedia": 0.8733375668525696, "Manosphere": 1.1549513339996338, "mC4": 0.9172928929328918, "Penn Treebank": 1.023613691329956, "RedPajama": 0.8071457743644714, "Twitter AAE": 2.506502389907837, "WikiText-103": 0.8607178330421448 } }, { "rung": "d1024", "step": 8403, "progress": 0.5500785545954439, "compute": 9.495006453216313e+19, "macro_loss": 2.97526478767395, "subset_bpb": { "4chan": 1.0826871395111084, "C4 100 domains": 0.9060752987861633, "C4 English": 0.940916121006012, "Dolma v1.5": 0.8849682211875916, "100 programming languages": 0.5648561716079712, "100 subreddits": 1.1115554571151733, "Falcon RefinedWeb": 0.9982771873474121, "Gab": 1.693160891532898, "M2D2 S2ORC": 0.8343638181686401, "M2D2 Wikipedia": 0.8675432205200195, "Manosphere": 1.1482501029968262, "mC4": 0.911216676235199, "Penn Treebank": 1.0267852544784546, "RedPajama": 0.80020672082901, "Twitter AAE": 2.4927761554718018, "WikiText-103": 0.8527136445045471 } }, { "rung": "d1024", "step": 9167, "progress": 0.6000916470280178, "compute": 1.035829158117743e+20, "macro_loss": 2.9497318267822266, "subset_bpb": { "4chan": 1.0767178535461426, "C4 100 domains": 0.8984890580177307, "C4 English": 0.9341745972633362, "Dolma v1.5": 0.8773166537284851, "100 programming languages": 0.5565683245658875, "100 subreddits": 1.1044402122497559, "Falcon RefinedWeb": 0.9907005429267883, "Gab": 1.675668716430664, "M2D2 S2ORC": 0.8273339867591858, "M2D2 Wikipedia": 0.8597126603126526, "Manosphere": 1.1421805620193481, "mC4": 0.9034002423286438, "Penn Treebank": 1.0108834505081177, "RedPajama": 0.7924314737319946, "Twitter AAE": 2.4758214950561523, "WikiText-103": 0.8450862765312195 } }, { "rung": "d1024", "step": 9931, "progress": 0.6501047394605918, "compute": 1.1221576709138547e+20, "macro_loss": 2.9277210235595703, "subset_bpb": { "4chan": 1.0693482160568237, "C4 100 domains": 0.8903468251228333, "C4 English": 0.9258646368980408, "Dolma v1.5": 0.870049774646759, "100 programming languages": 0.5496402978897095, "100 subreddits": 1.098796010017395, "Falcon RefinedWeb": 0.982456624507904, "Gab": 1.6666756868362427, "M2D2 S2ORC": 0.8201574683189392, "M2D2 Wikipedia": 0.8512833118438721, "Manosphere": 1.1355657577514648, "mC4": 0.8959702849388123, "Penn Treebank": 1.0053008794784546, "RedPajama": 0.7842409610748291, "Twitter AAE": 2.459467649459839, "WikiText-103": 0.8398590683937073 } }, { "rung": "d1024", "step": 10695, "progress": 0.7001178318931658, "compute": 1.2084861837099663e+20, "macro_loss": 2.907698392868042, "subset_bpb": { "4chan": 1.0599099397659302, "C4 100 domains": 0.8831467032432556, "C4 English": 0.9194046854972839, "Dolma v1.5": 0.8623067140579224, "100 programming languages": 0.5413411259651184, "100 subreddits": 1.0908821821212769, "Falcon RefinedWeb": 0.975348711013794, "Gab": 1.6637656688690186, "M2D2 S2ORC": 0.8128311634063721, "M2D2 Wikipedia": 0.8433969616889954, "Manosphere": 1.1286028623580933, "mC4": 0.8900366425514221, "Penn Treebank": 0.9977603554725647, "RedPajama": 0.7774479389190674, "Twitter AAE": 2.460221767425537, "WikiText-103": 0.8295368552207947 } }, { "rung": "d1024", "step": 11459, "progress": 0.7501309243257397, "compute": 1.294814696506078e+20, "macro_loss": 2.8838517665863037, "subset_bpb": { "4chan": 1.0543934106826782, "C4 100 domains": 0.8755173683166504, "C4 English": 0.9127313494682312, "Dolma v1.5": 0.8548634648323059, "100 programming languages": 0.5340237617492676, "100 subreddits": 1.0832463502883911, "Falcon RefinedWeb": 0.9677771329879761, "Gab": 1.6501868963241577, "M2D2 S2ORC": 0.8063364028930664, "M2D2 Wikipedia": 0.8356512188911438, "Manosphere": 1.120696783065796, "mC4": 0.8817587494850159, "Penn Treebank": 0.98636794090271, "RedPajama": 0.7690594792366028, "Twitter AAE": 2.4493038654327393, "WikiText-103": 0.8199109435081482 } }, { "rung": "d1024", "step": 12223, "progress": 0.8001440167583137, "compute": 1.3811432093021897e+20, "macro_loss": 2.8643991947174072, "subset_bpb": { "4chan": 1.0479998588562012, "C4 100 domains": 0.8682224154472351, "C4 English": 0.9057495594024658, "Dolma v1.5": 0.8479666113853455, "100 programming languages": 0.5276712775230408, "100 subreddits": 1.0777281522750854, "Falcon RefinedWeb": 0.960616946220398, "Gab": 1.6432195901870728, "M2D2 S2ORC": 0.8002006411552429, "M2D2 Wikipedia": 0.8277366757392883, "Manosphere": 1.1128534078598022, "mC4": 0.8739163279533386, "Penn Treebank": 0.9749721884727478, "RedPajama": 0.7624643445014954, "Twitter AAE": 2.4538497924804688, "WikiText-103": 0.8123905062675476 } }, { "rung": "d1024", "step": 12987, "progress": 0.8501571091908877, "compute": 1.4674717220983013e+20, "macro_loss": 2.8425564765930176, "subset_bpb": { "4chan": 1.039574146270752, "C4 100 domains": 0.8605895042419434, "C4 English": 0.8957265615463257, "Dolma v1.5": 0.8397854566574097, "100 programming languages": 0.5210819244384766, "100 subreddits": 1.0687824487686157, "Falcon RefinedWeb": 0.9512718319892883, "Gab": 1.635691523551941, "M2D2 S2ORC": 0.7959682941436768, "M2D2 Wikipedia": 0.8191460967063904, "Manosphere": 1.1074045896530151, "mC4": 0.8645539879798889, "Penn Treebank": 0.9688360691070557, "RedPajama": 0.7554923295974731, "Twitter AAE": 2.4514307975769043, "WikiText-103": 0.8024161458015442 } }, { "rung": "d1024", "step": 13751, "progress": 0.9001702016234616, "compute": 1.553800234894413e+20, "macro_loss": 2.8220372200012207, "subset_bpb": { "4chan": 1.0333900451660156, "C4 100 domains": 0.8527162671089172, "C4 English": 0.8879607915878296, "Dolma v1.5": 0.8327736854553223, "100 programming languages": 0.5140178203582764, "100 subreddits": 1.0624133348464966, "Falcon RefinedWeb": 0.9431416988372803, "Gab": 1.6276185512542725, "M2D2 S2ORC": 0.7895634174346924, "M2D2 Wikipedia": 0.8110822439193726, "Manosphere": 1.0993326902389526, "mC4": 0.8561424016952515, "Penn Treebank": 0.9582377076148987, "RedPajama": 0.7469741106033325, "Twitter AAE": 2.4580392837524414, "WikiText-103": 0.7946997880935669 } }, { "rung": "d1024", "step": 14515, "progress": 0.9501832940560356, "compute": 1.6401287476905247e+20, "macro_loss": 2.7989492416381836, "subset_bpb": { "4chan": 1.0277249813079834, "C4 100 domains": 0.8457332253456116, "C4 English": 0.8810853958129883, "Dolma v1.5": 0.8253218531608582, "100 programming languages": 0.5074296593666077, "100 subreddits": 1.0544719696044922, "Falcon RefinedWeb": 0.936080276966095, "Gab": 1.617185115814209, "M2D2 S2ORC": 0.7833781242370605, "M2D2 Wikipedia": 0.8034989833831787, "Manosphere": 1.0928376913070679, "mC4": 0.8490169644355774, "Penn Treebank": 0.9487255811691284, "RedPajama": 0.7399988770484924, "Twitter AAE": 2.4337852001190186, "WikiText-103": 0.7880728244781494 } }, { "rung": "d1024", "step": 15275, "progress": 0.9999345378371302, "compute": 1.7260052787442483e+20, "macro_loss": 2.780874729156494, "subset_bpb": { "4chan": 1.0213335752487183, "C4 100 domains": 0.8395844101905823, "C4 English": 0.8750015497207642, "Dolma v1.5": 0.8190394043922424, "100 programming languages": 0.5013558864593506, "100 subreddits": 1.0489264726638794, "Falcon RefinedWeb": 0.9295719861984253, "Gab": 1.6065263748168945, "M2D2 S2ORC": 0.7775632739067078, "M2D2 Wikipedia": 0.796610951423645, "Manosphere": 1.0870245695114136, "mC4": 0.8424513936042786, "Penn Treebank": 0.9412280917167664, "RedPajama": 0.7337654829025269, "Twitter AAE": 2.435474157333374, "WikiText-103": 0.7800092101097107 } }, { "rung": "d1536", "step": 755, "progress": 0.04990745637228979, "compute": 6.893845959495647e+19, "macro_loss": 3.1583311557769775, "subset_bpb": { "4chan": 1.1435256004333496, "C4 100 domains": 0.9669334292411804, "C4 English": 0.9991223216056824, "Dolma v1.5": 0.9483324885368347, "100 programming languages": 0.6252692937850952, "100 subreddits": 1.1696959733963013, "Falcon RefinedWeb": 1.060296654701233, "Gab": 1.7578245401382446, "M2D2 S2ORC": 0.8952763676643372, "M2D2 Wikipedia": 0.9325829744338989, "Manosphere": 1.2126104831695557, "mC4": 0.9730942249298096, "Penn Treebank": 1.1138008832931519, "RedPajama": 0.8666808009147644, "Twitter AAE": 2.497807264328003, "WikiText-103": 0.9323766827583313 } }, { "rung": "d1536", "step": 1511, "progress": 0.09988101533580117, "compute": 1.3796822840791951e+20, "macro_loss": 3.001892566680908, "subset_bpb": { "4chan": 1.0968739986419678, "C4 100 domains": 0.9142098426818848, "C4 English": 0.9496424794197083, "Dolma v1.5": 0.8956125378608704, "100 programming languages": 0.574367344379425, "100 subreddits": 1.1195017099380493, "Falcon RefinedWeb": 1.0080605745315552, "Gab": 1.6943588256835938, "M2D2 S2ORC": 0.843307614326477, "M2D2 Wikipedia": 0.8754068613052368, "Manosphere": 1.1587879657745361, "mC4": 0.9207177758216858, "Penn Treebank": 1.0332692861557007, "RedPajama": 0.8111599683761597, "Twitter AAE": 2.4722037315368652, "WikiText-103": 0.8675174713134766 } }, { "rung": "d1536", "step": 2267, "progress": 0.14985457429931254, "compute": 2.0699799722088255e+20, "macro_loss": 2.944444179534912, "subset_bpb": { "4chan": 1.0741405487060547, "C4 100 domains": 0.8952473402023315, "C4 English": 0.9311742186546326, "Dolma v1.5": 0.8757469058036804, "100 programming languages": 0.5569415092468262, "100 subreddits": 1.1039613485336304, "Falcon RefinedWeb": 0.9881086945533752, "Gab": 1.672205924987793, "M2D2 S2ORC": 0.8254367113113403, "M2D2 Wikipedia": 0.856462836265564, "Manosphere": 1.1396583318710327, "mC4": 0.9009880423545837, "Penn Treebank": 1.0029618740081787, "RedPajama": 0.7913020849227905, "Twitter AAE": 2.4649658203125, "WikiText-103": 0.8397175073623657 } }, { "rung": "d1536", "step": 3023, "progress": 0.1998281332628239, "compute": 2.760277660338456e+20, "macro_loss": 2.9032158851623535, "subset_bpb": { "4chan": 1.062366247177124, "C4 100 domains": 0.8832724094390869, "C4 English": 0.9194787740707397, "Dolma v1.5": 0.8632066249847412, "100 programming languages": 0.5420097708702087, "100 subreddits": 1.091638207435608, "Falcon RefinedWeb": 0.975221574306488, "Gab": 1.6523017883300781, "M2D2 S2ORC": 0.8147475719451904, "M2D2 Wikipedia": 0.8431525230407715, "Manosphere": 1.1264718770980835, "mC4": 0.886621356010437, "Penn Treebank": 0.9848235249519348, "RedPajama": 0.7787315845489502, "Twitter AAE": 2.4300713539123535, "WikiText-103": 0.8276329636573792 } }, { "rung": "d1536", "step": 3779, "progress": 0.24980169222633528, "compute": 3.450575348468086e+20, "macro_loss": 2.8789801597595215, "subset_bpb": { "4chan": 1.0595388412475586, "C4 100 domains": 0.8726962208747864, "C4 English": 0.9104818105697632, "Dolma v1.5": 0.852820873260498, "100 programming languages": 0.5354310870170593, "100 subreddits": 1.0813215970993042, "Falcon RefinedWeb": 0.9656778573989868, "Gab": 1.646085262298584, "M2D2 S2ORC": 0.8044227957725525, "M2D2 Wikipedia": 0.8326972126960754, "Manosphere": 1.1179172992706299, "mC4": 0.8801987767219543, "Penn Treebank": 0.9697641730308533, "RedPajama": 0.767880380153656, "Twitter AAE": 2.4384148120880127, "WikiText-103": 0.8194849491119385 } }, { "rung": "d1536", "step": 4535, "progress": 0.29977525118984666, "compute": 4.1408730365977166e+20, "macro_loss": 2.8418924808502197, "subset_bpb": { "4chan": 1.0403697490692139, "C4 100 domains": 0.8620900511741638, "C4 English": 0.9006562829017639, "Dolma v1.5": 0.8424891829490662, "100 programming languages": 0.5246390700340271, "100 subreddits": 1.0714362859725952, "Falcon RefinedWeb": 0.9547004699707031, "Gab": 1.6248024702072144, "M2D2 S2ORC": 0.7929751873016357, "M2D2 Wikipedia": 0.821484386920929, "Manosphere": 1.1088021993637085, "mC4": 0.8688307404518127, "Penn Treebank": 0.9539563655853271, "RedPajama": 0.7568314075469971, "Twitter AAE": 2.4169187545776367, "WikiText-103": 0.8019877672195435 } }, { "rung": "d1536", "step": 5291, "progress": 0.349748810153358, "compute": 4.831170724727347e+20, "macro_loss": 2.8284969329833984, "subset_bpb": { "4chan": 1.0342005491256714, "C4 100 domains": 0.8551620841026306, "C4 English": 0.8937745690345764, "Dolma v1.5": 0.8349674344062805, "100 programming languages": 0.5158284902572632, "100 subreddits": 1.0664533376693726, "Falcon RefinedWeb": 0.9487419128417969, "Gab": 1.6258288621902466, "M2D2 S2ORC": 0.7855128645896912, "M2D2 Wikipedia": 0.813857913017273, "Manosphere": 1.1012040376663208, "mC4": 0.863710880279541, "Penn Treebank": 0.9509348273277283, "RedPajama": 0.7510858774185181, "Twitter AAE": 2.4386696815490723, "WikiText-103": 0.7967999577522278 } }, { "rung": "d1536", "step": 6047, "progress": 0.39972236911686937, "compute": 5.521468412856977e+20, "macro_loss": 2.800361156463623, "subset_bpb": { "4chan": 1.0259586572647095, "C4 100 domains": 0.8469657897949219, "C4 English": 0.8853933811187744, "Dolma v1.5": 0.8267801403999329, "100 programming languages": 0.509177565574646, "100 subreddits": 1.0558385848999023, "Falcon RefinedWeb": 0.9394941329956055, "Gab": 1.6101371049880981, "M2D2 S2ORC": 0.7788002490997314, "M2D2 Wikipedia": 0.8056634664535522, "Manosphere": 1.0907716751098633, "mC4": 0.8536181449890137, "Penn Treebank": 0.9427475929260254, "RedPajama": 0.7424707412719727, "Twitter AAE": 2.413933753967285, "WikiText-103": 0.7870841026306152 } }, { "rung": "d1536", "step": 6803, "progress": 0.4496959280803807, "compute": 6.211766100986608e+20, "macro_loss": 2.7863872051239014, "subset_bpb": { "4chan": 1.0226624011993408, "C4 100 domains": 0.8413980007171631, "C4 English": 0.8806689381599426, "Dolma v1.5": 0.820453941822052, "100 programming languages": 0.503806471824646, "100 subreddits": 1.050873041152954, "Falcon RefinedWeb": 0.9344726800918579, "Gab": 1.610344409942627, "M2D2 S2ORC": 0.7721663117408752, "M2D2 Wikipedia": 0.7980539798736572, "Manosphere": 1.087365746498108, "mC4": 0.8495373725891113, "Penn Treebank": 0.9312788844108582, "RedPajama": 0.7359872460365295, "Twitter AAE": 2.4279699325561523, "WikiText-103": 0.7777612805366516 } }, { "rung": "d1536", "step": 7559, "progress": 0.49966948704389214, "compute": 6.902063789116238e+20, "macro_loss": 2.75604510307312, "subset_bpb": { "4chan": 1.0106366872787476, "C4 100 domains": 0.8329761028289795, "C4 English": 0.8718813061714172, "Dolma v1.5": 0.8139203786849976, "100 programming languages": 0.49611109495162964, "100 subreddits": 1.0439258813858032, "Falcon RefinedWeb": 0.9256200790405273, "Gab": 1.5899379253387451, "M2D2 S2ORC": 0.7665412425994873, "M2D2 Wikipedia": 0.7913109064102173, "Manosphere": 1.0796618461608887, "mC4": 0.8396135568618774, "Penn Treebank": 0.9202020168304443, "RedPajama": 0.7286281585693359, "Twitter AAE": 2.373278856277466, "WikiText-103": 0.7751572132110596 } }, { "rung": "d1536", "step": 8315, "progress": 0.5496430460074034, "compute": 7.592361477245868e+20, "macro_loss": 2.7345495223999023, "subset_bpb": { "4chan": 1.0061166286468506, "C4 100 domains": 0.8264982104301453, "C4 English": 0.8660121560096741, "Dolma v1.5": 0.8065928816795349, "100 programming languages": 0.4902944266796112, "100 subreddits": 1.0357168912887573, "Falcon RefinedWeb": 0.9185264110565186, "Gab": 1.5800487995147705, "M2D2 S2ORC": 0.7594812512397766, "M2D2 Wikipedia": 0.7835691571235657, "Manosphere": 1.0713025331497192, "mC4": 0.8336568474769592, "Penn Treebank": 0.9101556539535522, "RedPajama": 0.7211896777153015, "Twitter AAE": 2.370826482772827, "WikiText-103": 0.7620631456375122 } }, { "rung": "d1536", "step": 9071, "progress": 0.5996166049709148, "compute": 8.282659165375499e+20, "macro_loss": 2.714014768600464, "subset_bpb": { "4chan": 1.0005369186401367, "C4 100 domains": 0.8197023272514343, "C4 English": 0.860560953617096, "Dolma v1.5": 0.8002570271492004, "100 programming languages": 0.48589348793029785, "100 subreddits": 1.0292608737945557, "Falcon RefinedWeb": 0.912612795829773, "Gab": 1.5701264142990112, "M2D2 S2ORC": 0.7532532811164856, "M2D2 Wikipedia": 0.7773714661598206, "Manosphere": 1.064404845237732, "mC4": 0.8272955417633057, "Penn Treebank": 0.89668208360672, "RedPajama": 0.7150749564170837, "Twitter AAE": 2.366454601287842, "WikiText-103": 0.7498420476913452 } }, { "rung": "d1536", "step": 9827, "progress": 0.6495901639344263, "compute": 8.972956853505129e+20, "macro_loss": 2.6984610557556152, "subset_bpb": { "4chan": 0.9939626455307007, "C4 100 domains": 0.8126536011695862, "C4 English": 0.8541611433029175, "Dolma v1.5": 0.7935214638710022, "100 programming languages": 0.4797855615615845, "100 subreddits": 1.0240474939346313, "Falcon RefinedWeb": 0.9058167338371277, "Gab": 1.5687826871871948, "M2D2 S2ORC": 0.7466639876365662, "M2D2 Wikipedia": 0.7696113586425781, "Manosphere": 1.0590860843658447, "mC4": 0.8208391070365906, "Penn Treebank": 0.8871766328811646, "RedPajama": 0.7084018588066101, "Twitter AAE": 2.3801467418670654, "WikiText-103": 0.7448776960372925 } }, { "rung": "d1536", "step": 10583, "progress": 0.6995637228979376, "compute": 9.66325454163476e+20, "macro_loss": 2.6755783557891846, "subset_bpb": { "4chan": 0.9856505393981934, "C4 100 domains": 0.8061298727989197, "C4 English": 0.8473546504974365, "Dolma v1.5": 0.7868675589561462, "100 programming languages": 0.47431913018226624, "100 subreddits": 1.0167531967163086, "Falcon RefinedWeb": 0.8988666534423828, "Gab": 1.5534236431121826, "M2D2 S2ORC": 0.7394939661026001, "M2D2 Wikipedia": 0.7623195648193359, "Manosphere": 1.0532490015029907, "mC4": 0.8142158389091492, "Penn Treebank": 0.8804951906204224, "RedPajama": 0.7025076150894165, "Twitter AAE": 2.357056140899658, "WikiText-103": 0.7372839450836182 } }, { "rung": "d1536", "step": 11339, "progress": 0.749537281861449, "compute": 1.035355222976439e+21, "macro_loss": 2.6567752361297607, "subset_bpb": { "4chan": 0.9794723987579346, "C4 100 domains": 0.799557089805603, "C4 English": 0.8411847352981567, "Dolma v1.5": 0.7804861068725586, "100 programming languages": 0.46781304478645325, "100 subreddits": 1.010695457458496, "Falcon RefinedWeb": 0.8922535181045532, "Gab": 1.5452814102172852, "M2D2 S2ORC": 0.7344682812690735, "M2D2 Wikipedia": 0.7554432153701782, "Manosphere": 1.044956088066101, "mC4": 0.8072021007537842, "Penn Treebank": 0.8727163076400757, "RedPajama": 0.695805013179779, "Twitter AAE": 2.354499340057373, "WikiText-103": 0.7304077744483948 } }, { "rung": "d1536", "step": 12095, "progress": 0.7995108408249604, "compute": 1.104384991789402e+21, "macro_loss": 2.638200283050537, "subset_bpb": { "4chan": 0.974311888217926, "C4 100 domains": 0.7928212285041809, "C4 English": 0.8349142074584961, "Dolma v1.5": 0.7737289071083069, "100 programming languages": 0.46062272787094116, "100 subreddits": 1.0052685737609863, "Falcon RefinedWeb": 0.8852970004081726, "Gab": 1.540278434753418, "M2D2 S2ORC": 0.728196382522583, "M2D2 Wikipedia": 0.7483054399490356, "Manosphere": 1.0390961170196533, "mC4": 0.799950361251831, "Penn Treebank": 0.865463137626648, "RedPajama": 0.6884734034538269, "Twitter AAE": 2.350921630859375, "WikiText-103": 0.7224705219268799 } }, { "rung": "d1536", "step": 12851, "progress": 0.8494843997884717, "compute": 1.1734147606023651e+21, "macro_loss": 2.619727373123169, "subset_bpb": { "4chan": 0.9659073948860168, "C4 100 domains": 0.785959780216217, "C4 English": 0.8253793716430664, "Dolma v1.5": 0.7666188478469849, "100 programming languages": 0.4553206264972687, "100 subreddits": 0.9967175126075745, "Falcon RefinedWeb": 0.8771544098854065, "Gab": 1.5378693342208862, "M2D2 S2ORC": 0.7249725461006165, "M2D2 Wikipedia": 0.740237295627594, "Manosphere": 1.032678246498108, "mC4": 0.7901648283004761, "Penn Treebank": 0.8604143857955933, "RedPajama": 0.6820822358131409, "Twitter AAE": 2.356369972229004, "WikiText-103": 0.7139933705329895 } }, { "rung": "d1536", "step": 13607, "progress": 0.8994579587519831, "compute": 1.242444529415328e+21, "macro_loss": 2.599482536315918, "subset_bpb": { "4chan": 0.9588054418563843, "C4 100 domains": 0.7792117595672607, "C4 English": 0.8187023997306824, "Dolma v1.5": 0.7597551345825195, "100 programming languages": 0.44944295287132263, "100 subreddits": 0.9899399876594543, "Falcon RefinedWeb": 0.8697390556335449, "Gab": 1.5287152528762817, "M2D2 S2ORC": 0.719203770160675, "M2D2 Wikipedia": 0.7330495119094849, "Manosphere": 1.0252312421798706, "mC4": 0.7827052474021912, "Penn Treebank": 0.8522951006889343, "RedPajama": 0.6760824918746948, "Twitter AAE": 2.3530826568603516, "WikiText-103": 0.7042036056518555 } }, { "rung": "d1536", "step": 14363, "progress": 0.9494315177154945, "compute": 1.3114742982282911e+21, "macro_loss": 2.579882860183716, "subset_bpb": { "4chan": 0.9532797932624817, "C4 100 domains": 0.7727611660957336, "C4 English": 0.8119158744812012, "Dolma v1.5": 0.7528673410415649, "100 programming languages": 0.4424571394920349, "100 subreddits": 0.9839339852333069, "Falcon RefinedWeb": 0.8630123734474182, "Gab": 1.5213000774383545, "M2D2 S2ORC": 0.7131468057632446, "M2D2 Wikipedia": 0.7260826230049133, "Manosphere": 1.0196284055709839, "mC4": 0.7745689153671265, "Penn Treebank": 0.8452649712562561, "RedPajama": 0.6691806316375732, "Twitter AAE": 2.344827651977539, "WikiText-103": 0.6964027285575867 } }, { "rung": "d1536", "step": 15119, "progress": 0.9994050766790058, "compute": 1.3805040670412542e+21, "macro_loss": 2.5638508796691895, "subset_bpb": { "4chan": 0.9498159289360046, "C4 100 domains": 0.7672283053398132, "C4 English": 0.8065240383148193, "Dolma v1.5": 0.7473235130310059, "100 programming languages": 0.4375442862510681, "100 subreddits": 0.9790083765983582, "Falcon RefinedWeb": 0.8573029637336731, "Gab": 1.5134024620056152, "M2D2 S2ORC": 0.7080243229866028, "M2D2 Wikipedia": 0.7198999524116516, "Manosphere": 1.0148699283599854, "mC4": 0.7687363624572754, "Penn Treebank": 0.8393762707710266, "RedPajama": 0.6636946797370911, "Twitter AAE": 2.338204860687256, "WikiText-103": 0.690193235874176 } }, { "rung": "d1536", "step": 15127, "progress": 0.9999338974087785, "compute": 1.3812345407853067e+21, "macro_loss": 2.5637738704681396, "subset_bpb": { "4chan": 0.9493623971939087, "C4 100 domains": 0.767145574092865, "C4 English": 0.8064409494400024, "Dolma v1.5": 0.7472224831581116, "100 programming languages": 0.4370846748352051, "100 subreddits": 0.9789345264434814, "Falcon RefinedWeb": 0.8571977019309998, "Gab": 1.5129204988479614, "M2D2 S2ORC": 0.708041250705719, "M2D2 Wikipedia": 0.7198629975318909, "Manosphere": 1.014733076095581, "mC4": 0.7684661746025085, "Penn Treebank": 0.8395755887031555, "RedPajama": 0.6636402606964111, "Twitter AAE": 2.3402576446533203, "WikiText-103": 0.6902027726173401 } }, { "rung": "d2048", "step": 1003, "progress": 0.04997010761259466, "compute": 3.8685932605118664e+20, "macro_loss": 2.9312033653259277, "subset_bpb": { "4chan": 1.072113275527954, "C4 100 domains": 0.8904330730438232, "C4 English": 0.926417887210846, "Dolma v1.5": 0.8722180128097534, "100 programming languages": 0.5507325530052185, "100 subreddits": 1.101016879081726, "Falcon RefinedWeb": 0.9819079041481018, "Gab": 1.6623671054840088, "M2D2 S2ORC": 0.8196195960044861, "M2D2 Wikipedia": 0.8508392572402954, "Manosphere": 1.1414910554885864, "mC4": 0.8959046602249146, "Penn Treebank": 1.0028200149536133, "RedPajama": 0.7890208959579468, "Twitter AAE": 2.4431447982788086, "WikiText-103": 0.8410398364067078 } }, { "rung": "d2048", "step": 2007, "progress": 0.09999003587086489, "compute": 7.741043543217663e+20, "macro_loss": 2.802788257598877, "subset_bpb": { "4chan": 1.0276437997817993, "C4 100 domains": 0.8495497703552246, "C4 English": 0.887365996837616, "Dolma v1.5": 0.8295342922210693, "100 programming languages": 0.5072946548461914, "100 subreddits": 1.0597163438796997, "Falcon RefinedWeb": 0.9404405951499939, "Gab": 1.6120200157165527, "M2D2 S2ORC": 0.7792518734931946, "M2D2 Wikipedia": 0.806933581829071, "Manosphere": 1.0962913036346436, "mC4": 0.8544517755508423, "Penn Treebank": 0.9424226880073547, "RedPajama": 0.7464707493782043, "Twitter AAE": 2.404188394546509, "WikiText-103": 0.7837916016578674 } }, { "rung": "d2048", "step": 3011, "progress": 0.15000996412913511, "compute": 1.1613493825923459e+21, "macro_loss": 2.7470338344573975, "subset_bpb": { "4chan": 1.0104753971099854, "C4 100 domains": 0.8315423727035522, "C4 English": 0.8712862730026245, "Dolma v1.5": 0.8124889135360718, "100 programming languages": 0.4922250211238861, "100 subreddits": 1.0416253805160522, "Falcon RefinedWeb": 0.9232621192932129, "Gab": 1.5856437683105469, "M2D2 S2ORC": 0.7635026574134827, "M2D2 Wikipedia": 0.7886809706687927, "Manosphere": 1.079684853553772, "mC4": 0.8371813893318176, "Penn Treebank": 0.9181463122367859, "RedPajama": 0.7293550372123718, "Twitter AAE": 2.363478660583496, "WikiText-103": 0.7615338563919067 } }, { "rung": "d2048", "step": 4015, "progress": 0.20002989238740534, "compute": 1.5485944108629256e+21, "macro_loss": 2.710448741912842, "subset_bpb": { "4chan": 0.9941623210906982, "C4 100 domains": 0.8196429014205933, "C4 English": 0.8609788417816162, "Dolma v1.5": 0.8008083701133728, "100 programming languages": 0.48087042570114136, "100 subreddits": 1.0289541482925415, "Falcon RefinedWeb": 0.9116232395172119, "Gab": 1.569047212600708, "M2D2 S2ORC": 0.7507500052452087, "M2D2 Wikipedia": 0.7764437198638916, "Manosphere": 1.0639947652816772, "mC4": 0.8251298666000366, "Penn Treebank": 0.8954340219497681, "RedPajama": 0.7179815769195557, "Twitter AAE": 2.3583292961120605, "WikiText-103": 0.7534505724906921 } }, { "rung": "d2048", "step": 5019, "progress": 0.25004982064567555, "compute": 1.9358394391335052e+21, "macro_loss": 2.6807942390441895, "subset_bpb": { "4chan": 0.9859506487846375, "C4 100 domains": 0.8094303607940674, "C4 English": 0.8511794209480286, "Dolma v1.5": 0.7902066707611084, "100 programming languages": 0.476570725440979, "100 subreddits": 1.0188608169555664, "Falcon RefinedWeb": 0.901584804058075, "Gab": 1.5574133396148682, "M2D2 S2ORC": 0.7408965826034546, "M2D2 Wikipedia": 0.7653893232345581, "Manosphere": 1.0546116828918457, "mC4": 0.8157801628112793, "Penn Treebank": 0.8759937882423401, "RedPajama": 0.7077699303627014, "Twitter AAE": 2.3653557300567627, "WikiText-103": 0.7334079742431641 } }, { "rung": "d2048", "step": 6023, "progress": 0.3000697489039458, "compute": 2.323084467404085e+21, "macro_loss": 2.6612348556518555, "subset_bpb": { "4chan": 0.9790515899658203, "C4 100 domains": 0.8021419048309326, "C4 English": 0.8442353010177612, "Dolma v1.5": 0.7840926647186279, "100 programming languages": 0.46913859248161316, "100 subreddits": 1.0132005214691162, "Falcon RefinedWeb": 0.8943842649459839, "Gab": 1.5554784536361694, "M2D2 S2ORC": 0.7355581521987915, "M2D2 Wikipedia": 0.7587690949440002, "Manosphere": 1.0462614297866821, "mC4": 0.8076685070991516, "Penn Treebank": 0.8665377497673035, "RedPajama": 0.701909065246582, "Twitter AAE": 2.3529205322265625, "WikiText-103": 0.7290784120559692 } }, { "rung": "d2048", "step": 7027, "progress": 0.350089677162216, "compute": 2.7103294956746645e+21, "macro_loss": 2.643836498260498, "subset_bpb": { "4chan": 0.9746859669685364, "C4 100 domains": 0.7959907054901123, "C4 English": 0.8382603526115417, "Dolma v1.5": 0.7776228189468384, "100 programming languages": 0.46485570073127747, "100 subreddits": 1.0097970962524414, "Falcon RefinedWeb": 0.8876832127571106, "Gab": 1.550722360610962, "M2D2 S2ORC": 0.7298161387443542, "M2D2 Wikipedia": 0.7509550452232361, "Manosphere": 1.0418115854263306, "mC4": 0.8014534115791321, "Penn Treebank": 0.8628266453742981, "RedPajama": 0.6945967078208923, "Twitter AAE": 2.341611623764038, "WikiText-103": 0.7201178669929504 } }, { "rung": "d2048", "step": 8031, "progress": 0.40010960542048624, "compute": 3.097574523945244e+21, "macro_loss": 2.6150641441345215, "subset_bpb": { "4chan": 0.967901885509491, "C4 100 domains": 0.7890822291374207, "C4 English": 0.8302788138389587, "Dolma v1.5": 0.7700214385986328, "100 programming languages": 0.45563825964927673, "100 subreddits": 0.999732494354248, "Falcon RefinedWeb": 0.8795512318611145, "Gab": 1.5335744619369507, "M2D2 S2ORC": 0.7213730812072754, "M2D2 Wikipedia": 0.7435262799263, "Manosphere": 1.0327200889587402, "mC4": 0.7935119271278381, "Penn Treebank": 0.8435006141662598, "RedPajama": 0.6870576739311218, "Twitter AAE": 2.3174631595611572, "WikiText-103": 0.7127164006233215 } }, { "rung": "d2048", "step": 9035, "progress": 0.45012953367875647, "compute": 3.484819552215824e+21, "macro_loss": 2.6094987392425537, "subset_bpb": { "4chan": 0.9650702476501465, "C4 100 domains": 0.7830730080604553, "C4 English": 0.8261458873748779, "Dolma v1.5": 0.7645055055618286, "100 programming languages": 0.4531325101852417, "100 subreddits": 0.9964165687561035, "Falcon RefinedWeb": 0.875457227230072, "Gab": 1.5352201461791992, "M2D2 S2ORC": 0.7161030769348145, "M2D2 Wikipedia": 0.7367092370986938, "Manosphere": 1.028425693511963, "mC4": 0.7896668314933777, "Penn Treebank": 0.8446135520935059, "RedPajama": 0.6820812225341797, "Twitter AAE": 2.3593623638153076, "WikiText-103": 0.7055559754371643 } }, { "rung": "d2048", "step": 10039, "progress": 0.5001494619370267, "compute": 3.8720645804864035e+21, "macro_loss": 2.590702533721924, "subset_bpb": { "4chan": 0.9562656879425049, "C4 100 domains": 0.7763481736183167, "C4 English": 0.8193398118019104, "Dolma v1.5": 0.7580492496490479, "100 programming languages": 0.4458492398262024, "100 subreddits": 0.9891542196273804, "Falcon RefinedWeb": 0.8684014678001404, "Gab": 1.5274745225906372, "M2D2 S2ORC": 0.7098261713981628, "M2D2 Wikipedia": 0.7307369112968445, "Manosphere": 1.0228337049484253, "mC4": 0.7818675637245178, "Penn Treebank": 0.8364415764808655, "RedPajama": 0.6751725077629089, "Twitter AAE": 2.361626148223877, "WikiText-103": 0.6994859576225281 } }, { "rung": "d2048", "step": 11043, "progress": 0.550169390195297, "compute": 4.259309608756983e+21, "macro_loss": 2.5618793964385986, "subset_bpb": { "4chan": 0.9474497437477112, "C4 100 domains": 0.7706407308578491, "C4 English": 0.8139439821243286, "Dolma v1.5": 0.7516139149665833, "100 programming languages": 0.4409319758415222, "100 subreddits": 0.9807339310646057, "Falcon RefinedWeb": 0.8618955016136169, "Gab": 1.5089290142059326, "M2D2 S2ORC": 0.7046525478363037, "M2D2 Wikipedia": 0.7245576977729797, "Manosphere": 1.0140336751937866, "mC4": 0.7753936648368835, "Penn Treebank": 0.8264612555503845, "RedPajama": 0.6694921851158142, "Twitter AAE": 2.296968460083008, "WikiText-103": 0.6919926404953003 } }, { "rung": "d2048", "step": 12047, "progress": 0.6001893184535672, "compute": 4.646554637027563e+21, "macro_loss": 2.543623447418213, "subset_bpb": { "4chan": 0.9415086507797241, "C4 100 domains": 0.7651631832122803, "C4 English": 0.8080764412879944, "Dolma v1.5": 0.7455806136131287, "100 programming languages": 0.43494483828544617, "100 subreddits": 0.9774448275566101, "Falcon RefinedWeb": 0.8558962345123291, "Gab": 1.5000275373458862, "M2D2 S2ORC": 0.6981356143951416, "M2D2 Wikipedia": 0.7178562879562378, "Manosphere": 1.0083001852035522, "mC4": 0.7685183882713318, "Penn Treebank": 0.8192719221115112, "RedPajama": 0.6629815101623535, "Twitter AAE": 2.2912211418151855, "WikiText-103": 0.6829707026481628 } }, { "rung": "d2048", "step": 13051, "progress": 0.6502092467118374, "compute": 5.033799665298142e+21, "macro_loss": 2.527618408203125, "subset_bpb": { "4chan": 0.9354525804519653, "C4 100 domains": 0.7581421732902527, "C4 English": 0.8017220497131348, "Dolma v1.5": 0.7395548224449158, "100 programming languages": 0.42879945039749146, "100 subreddits": 0.9699209332466125, "Falcon RefinedWeb": 0.84893798828125, "Gab": 1.4990077018737793, "M2D2 S2ORC": 0.6926255822181702, "M2D2 Wikipedia": 0.7111137509346008, "Manosphere": 1.0028313398361206, "mC4": 0.7617015838623047, "Penn Treebank": 0.8112249374389648, "RedPajama": 0.6573758125305176, "Twitter AAE": 2.3016390800476074, "WikiText-103": 0.6749602556228638 } }, { "rung": "d2048", "step": 14055, "progress": 0.7002291749701076, "compute": 5.421044693568722e+21, "macro_loss": 2.5085854530334473, "subset_bpb": { "4chan": 0.9304894208908081, "C4 100 domains": 0.7516301274299622, "C4 English": 0.7954370975494385, "Dolma v1.5": 0.7330986261367798, "100 programming languages": 0.4221549928188324, "100 subreddits": 0.9634693264961243, "Falcon RefinedWeb": 0.8424212336540222, "Gab": 1.4856958389282227, "M2D2 S2ORC": 0.686876654624939, "M2D2 Wikipedia": 0.70439612865448, "Manosphere": 0.9958490133285522, "mC4": 0.7550739645957947, "Penn Treebank": 0.8082111477851868, "RedPajama": 0.6503385901451111, "Twitter AAE": 2.293320417404175, "WikiText-103": 0.6687796115875244 } }, { "rung": "d2048", "step": 15059, "progress": 0.7502491032283778, "compute": 5.808289721839302e+21, "macro_loss": 2.4904448986053467, "subset_bpb": { "4chan": 0.9242264628410339, "C4 100 domains": 0.7455265522003174, "C4 English": 0.7894375324249268, "Dolma v1.5": 0.7270591259002686, "100 programming languages": 0.41740840673446655, "100 subreddits": 0.9585963487625122, "Falcon RefinedWeb": 0.8358680009841919, "Gab": 1.4757707118988037, "M2D2 S2ORC": 0.6810687184333801, "M2D2 Wikipedia": 0.6977661848068237, "Manosphere": 0.9904647469520569, "mC4": 0.7489031553268433, "Penn Treebank": 0.7982411980628967, "RedPajama": 0.6450161337852478, "Twitter AAE": 2.288945198059082, "WikiText-103": 0.6623416543006897 } }, { "rung": "d2048", "step": 16063, "progress": 0.8002690314866481, "compute": 6.195534750109881e+21, "macro_loss": 2.474504232406616, "subset_bpb": { "4chan": 0.9191761612892151, "C4 100 domains": 0.7396953701972961, "C4 English": 0.7834416627883911, "Dolma v1.5": 0.7209616899490356, "100 programming languages": 0.4116165041923523, "100 subreddits": 0.9517097473144531, "Falcon RefinedWeb": 0.829352855682373, "Gab": 1.4758743047714233, "M2D2 S2ORC": 0.6756245493888855, "M2D2 Wikipedia": 0.6911993026733398, "Manosphere": 0.9848760962486267, "mC4": 0.7417214512825012, "Penn Treebank": 0.7904568314552307, "RedPajama": 0.639616847038269, "Twitter AAE": 2.2936666011810303, "WikiText-103": 0.6536816358566284 } } ], "subsets": { "4chan": { "calibration": { "intercept": 0.1153492678467611, "slope": 0.32542695123046494, "r_squared": 0.9993840407834024, "rmse": 0.002313939116727117, "max_abs_residual": 0.009324457127483532, "n": 77 }, "hero_forecast_bpb": 0.7788247240089671, "sensitivity_forecasts": { "all_observations": 0.7788247240089671, "progress_ge_60_percent": 0.7763465239608289, "last_observation_per_rung": 0.7737120191460947 }, "sensitivity_range": [ 0.7737120191460947, 0.7788247240089671 ], "conversion_bootstrap_95_percent": [ 0.77360993764319, 0.7842233746478353 ], "current_hero_measured_bpb": 0.8472199440002441, "current_hero_predicted_bpb": 0.8504062952948824, "qwen_measured_bpb": 0.88278930596034, "qwen_training_compute": 6.48e+23, "endpoints": [ { "rung": "d768", "full_compute": 2.649679880270119e+19, "measured_compute": 2.6494478592648413e+19, "measured_progress": 0.9999124343257443, "measured_bpb": 1.1000752449035645, "measured_macro_loss": 3.015353202819824 }, { "rung": "d1024", "full_compute": 1.7261182741798453e+20, "measured_compute": 1.7260052787442483e+20, "measured_progress": 0.9999345378371302, "measured_bpb": 1.0213335752487183, "measured_macro_loss": 2.780874729156494 }, { "rung": "d1536", "full_compute": 1.3813258500033133e+21, "measured_compute": 1.3812345407853067e+21, "measured_progress": 0.9999338974087785, "measured_bpb": 0.9493623971939087, "measured_macro_loss": 2.5637738704681396 }, { "rung": "d2048", "full_compute": 7.741814947656449e+21, "measured_compute": 6.195534750109881e+21, "measured_progress": 0.8002690314866481, "measured_bpb": 0.9191761612892151, "measured_macro_loss": 2.474504232406616, "forecast_full_bpb": 0.899207809096452 } ], "nemotron_nano_measured_bpb": 0.8529030813768576, "nemotron_nano_training_compute": 5.25e+23, "nemotron_super_measured_bpb": 0.793467863723923, "nemotron_super_training_compute": 1.8e+24 }, "C4 100 domains": { "calibration": { "intercept": -0.09147791261476923, "slope": 0.3354014491429544, "r_squared": 0.9997339802398767, "rmse": 0.0015669974612640195, "max_abs_residual": 0.0051407932372136145, "n": 77 }, "hero_forecast_bpb": 0.5923333962709292, "sensitivity_forecasts": { "all_observations": 0.5923333962709292, "progress_ge_60_percent": 0.593613413887432, "last_observation_per_rung": 0.5938812933943053 }, "sensitivity_range": [ 0.5923333962709292, 0.5938812933943053 ], "conversion_bootstrap_95_percent": [ 0.5880596657319319, 0.5952388460392045 ], "current_hero_measured_bpb": 0.6743637919425964, "current_hero_predicted_bpb": 0.666108978319212, "qwen_measured_bpb": 0.7045068602207951, "qwen_training_compute": 6.48e+23, "endpoints": [ { "rung": "d768", "full_compute": 2.649679880270119e+19, "measured_compute": 2.6494478592648413e+19, "measured_progress": 0.9999124343257443, "measured_bpb": 0.9186555743217468, "measured_macro_loss": 3.015353202819824 }, { "rung": "d1024", "full_compute": 1.7261182741798453e+20, "measured_compute": 1.7260052787442483e+20, "measured_progress": 0.9999345378371302, "measured_bpb": 0.8395844101905823, "measured_macro_loss": 2.780874729156494 }, { "rung": "d1536", "full_compute": 1.3813258500033133e+21, "measured_compute": 1.3812345407853067e+21, "measured_progress": 0.9999338974087785, "measured_bpb": 0.767145574092865, "measured_macro_loss": 2.5637738704681396 }, { "rung": "d2048", "full_compute": 7.741814947656449e+21, "measured_compute": 6.195534750109881e+21, "measured_progress": 0.8002690314866481, "measured_bpb": 0.7396953701972961, "measured_macro_loss": 2.474504232406616, "forecast_full_bpb": 0.7164062827911596 } ], "nemotron_nano_measured_bpb": 0.7004211698724362, "nemotron_nano_training_compute": 5.25e+23, "nemotron_super_measured_bpb": 0.633082710860957, "nemotron_super_training_compute": 1.8e+24 }, "C4 English": { "calibration": { "intercept": 0.005171275732166117, "slope": 0.31436623846707656, "r_squared": 0.9993910412804824, "rmse": 0.0022225458833503037, "max_abs_residual": 0.004801975109620882, "n": 77 }, "hero_forecast_bpb": 0.6460963210620155, "sensitivity_forecasts": { "all_observations": 0.6460963210620155, "progress_ge_60_percent": 0.6458557659327043, "last_observation_per_rung": 0.646500625688442 }, "sensitivity_range": [ 0.6458557659327043, 0.646500625688442 ], "conversion_bootstrap_95_percent": [ 0.6403441114927648, 0.6491998194553912 ], "current_hero_measured_bpb": 0.7177862524986267, "current_hero_predicted_bpb": 0.7152449555670285, "qwen_measured_bpb": 0.7549489519312605, "qwen_training_compute": 6.48e+23, "endpoints": [ { "rung": "d768", "full_compute": 2.649679880270119e+19, "measured_compute": 2.6494478592648413e+19, "measured_progress": 0.9999124343257443, "measured_bpb": 0.9491547346115112, "measured_macro_loss": 3.015353202819824 }, { "rung": "d1024", "full_compute": 1.7261182741798453e+20, "measured_compute": 1.7260052787442483e+20, "measured_progress": 0.9999345378371302, "measured_bpb": 0.8750015497207642, "measured_macro_loss": 2.780874729156494 }, { "rung": "d1536", "full_compute": 1.3813258500033133e+21, "measured_compute": 1.3812345407853067e+21, "measured_progress": 0.9999338974087785, "measured_bpb": 0.8064409494400024, "measured_macro_loss": 2.5637738704681396 }, { "rung": "d2048", "full_compute": 7.741814947656449e+21, "measured_compute": 6.195534750109881e+21, "measured_progress": 0.8002690314866481, "measured_bpb": 0.7834416627883911, "measured_macro_loss": 2.474504232406616, "forecast_full_bpb": 0.7623877882896747 } ], "nemotron_nano_measured_bpb": 0.747164483671727, "nemotron_nano_training_compute": 5.25e+23, "nemotron_super_measured_bpb": 0.675213142735745, "nemotron_super_training_compute": 1.8e+24 }, "Dolma v1.5": { "calibration": { "intercept": -0.09877128502855785, "slope": 0.330873406906626, "r_squared": 0.9996372297493297, "rmse": 0.001805282286814752, "max_abs_residual": 0.0045787142873159326, "n": 77 }, "hero_forecast_bpb": 0.5758083210835675, "sensitivity_forecasts": { "all_observations": 0.5758083210835675, "progress_ge_60_percent": 0.5779831851772406, "last_observation_per_rung": 0.577872574482124 }, "sensitivity_range": [ 0.5758083210835675, 0.5779831851772406 ], "conversion_bootstrap_95_percent": [ 0.5702996743880209, 0.5783536773931269 ], "current_hero_measured_bpb": 0.650954008102417, "current_hero_predicted_bpb": 0.6485879057908193, "qwen_measured_bpb": 0.6582074468864063, "qwen_training_compute": 6.48e+23, "endpoints": [ { "rung": "d768", "full_compute": 2.649679880270119e+19, "measured_compute": 2.6494478592648413e+19, "measured_progress": 0.9999124343257443, "measured_bpb": 0.8959868550300598, "measured_macro_loss": 3.015353202819824 }, { "rung": "d1024", "full_compute": 1.7261182741798453e+20, "measured_compute": 1.7260052787442483e+20, "measured_progress": 0.9999345378371302, "measured_bpb": 0.8190394043922424, "measured_macro_loss": 2.780874729156494 }, { "rung": "d1536", "full_compute": 1.3813258500033133e+21, "measured_compute": 1.3812345407853067e+21, "measured_progress": 0.9999338974087785, "measured_bpb": 0.7472224831581116, "measured_macro_loss": 2.5637738704681396 }, { "rung": "d2048", "full_compute": 7.741814947656449e+21, "measured_compute": 6.195534750109881e+21, "measured_progress": 0.8002690314866481, "measured_bpb": 0.7209616899490356, "measured_macro_loss": 2.474504232406616, "forecast_full_bpb": 0.6982061782561837 } ], "nemotron_nano_measured_bpb": 0.6621254426290294, "nemotron_nano_training_compute": 5.25e+23, "nemotron_super_measured_bpb": 0.593682537587207, "nemotron_super_training_compute": 1.8e+24 }, "100 programming languages": { "calibration": { "intercept": -0.39093683571087373, "slope": 0.32228269950456045, "r_squared": 0.9983106497294301, "rmse": 0.0037971052019684513, "max_abs_residual": 0.017786240009500354, "n": 77 }, "hero_forecast_bpb": 0.26612816845820303, "sensitivity_forecasts": { "all_observations": 0.26612816845820303, "progress_ge_60_percent": 0.27575691961623355, "last_observation_per_rung": 0.2773677139310197 }, "sensitivity_range": [ 0.26612816845820303, 0.2773677139310197 ], "conversion_bootstrap_95_percent": [ 0.24755124081214486, 0.27768336051943254 ], "current_hero_measured_bpb": 0.3567442297935486, "current_hero_predicted_bpb": 0.3370181237663769, "qwen_measured_bpb": 0.33941854758748374, "qwen_training_compute": 6.48e+23, "endpoints": [ { "rung": "d768", "full_compute": 2.649679880270119e+19, "measured_compute": 2.6494478592648413e+19, "measured_progress": 0.9999124343257443, "measured_bpb": 0.5766160488128662, "measured_macro_loss": 3.015353202819824 }, { "rung": "d1024", "full_compute": 1.7261182741798453e+20, "measured_compute": 1.7260052787442483e+20, "measured_progress": 0.9999345378371302, "measured_bpb": 0.5013558864593506, "measured_macro_loss": 2.780874729156494 }, { "rung": "d1536", "full_compute": 1.3813258500033133e+21, "measured_compute": 1.3812345407853067e+21, "measured_progress": 0.9999338974087785, "measured_bpb": 0.4370846748352051, "measured_macro_loss": 2.5637738704681396 }, { "rung": "d2048", "full_compute": 7.741814947656449e+21, "measured_compute": 6.195534750109881e+21, "measured_progress": 0.8002690314866481, "measured_bpb": 0.4116165041923523, "measured_macro_loss": 2.474504232406616, "forecast_full_bpb": 0.3853481208622244 } ], "nemotron_nano_measured_bpb": 0.39262551987799477, "nemotron_nano_training_compute": 5.25e+23, "nemotron_super_measured_bpb": 0.323534044276444, "nemotron_super_training_compute": 1.8e+24 }, "100 subreddits": { "calibration": { "intercept": 0.17781645698796283, "slope": 0.31380267447990495, "r_squared": 0.9994448819572106, "rmse": 0.002118158839065766, "max_abs_residual": 0.007606634108547228, "n": 77 }, "hero_forecast_bpb": 0.8175925167403992, "sensitivity_forecasts": { "all_observations": 0.8175925167403992, "progress_ge_60_percent": 0.8129504476557977, "last_observation_per_rung": 0.8116314854146085 }, "sensitivity_range": [ 0.8116314854146085, 0.8175925167403992 ], "conversion_bootstrap_95_percent": [ 0.8127871772042259, 0.8231809228678194 ], "current_hero_measured_bpb": 0.8852283954620361, "current_hero_predicted_bpb": 0.8866171885693974, "qwen_measured_bpb": 0.9136682068914669, "qwen_training_compute": 6.48e+23, "endpoints": [ { "rung": "d768", "full_compute": 2.649679880270119e+19, "measured_compute": 2.6494478592648413e+19, "measured_progress": 0.9999124343257443, "measured_bpb": 1.1243383884429932, "measured_macro_loss": 3.015353202819824 }, { "rung": "d1024", "full_compute": 1.7261182741798453e+20, "measured_compute": 1.7260052787442483e+20, "measured_progress": 0.9999345378371302, "measured_bpb": 1.0489264726638794, "measured_macro_loss": 2.780874729156494 }, { "rung": "d1536", "full_compute": 1.3813258500033133e+21, "measured_compute": 1.3812345407853067e+21, "measured_progress": 0.9999338974087785, "measured_bpb": 0.9789345264434814, "measured_macro_loss": 2.5637738704681396 }, { "rung": "d2048", "full_compute": 7.741814947656449e+21, "measured_compute": 6.195534750109881e+21, "measured_progress": 0.8002690314866481, "measured_bpb": 0.9517097473144531, "measured_macro_loss": 2.474504232406616, "forecast_full_bpb": 0.9336755083910868 } ], "nemotron_nano_measured_bpb": 0.9192233774182177, "nemotron_nano_training_compute": 5.25e+23, "nemotron_super_measured_bpb": 0.841506086255642, "nemotron_super_training_compute": 1.8e+24 }, "Falcon RefinedWeb": { "calibration": { "intercept": 0.0008679104519221797, "slope": 0.33516056503809266, "r_squared": 0.9997259502077749, "rmse": 0.0015893363124070445, "max_abs_residual": 0.0035618008287792158, "n": 77 }, "hero_forecast_bpb": 0.684188108534563, "sensitivity_forecasts": { "all_observations": 0.684188108534563, "progress_ge_60_percent": 0.6837703663261472, "last_observation_per_rung": 0.6832534360796422 }, "sensitivity_range": [ 0.6832534360796422, 0.684188108534563 ], "conversion_bootstrap_95_percent": [ 0.6803511004453471, 0.6856771700513085 ], "current_hero_measured_bpb": 0.7534131407737732, "current_hero_predicted_bpb": 0.7579107052272734, "qwen_measured_bpb": 0.7787743733125458, "qwen_training_compute": 6.48e+23, "endpoints": [ { "rung": "d768", "full_compute": 2.649679880270119e+19, "measured_compute": 2.6494478592648413e+19, "measured_progress": 0.9999124343257443, "measured_bpb": 1.0089563131332397, "measured_macro_loss": 3.015353202819824 }, { "rung": "d1024", "full_compute": 1.7261182741798453e+20, "measured_compute": 1.7260052787442483e+20, "measured_progress": 0.9999345378371302, "measured_bpb": 0.9295719861984253, "measured_macro_loss": 2.780874729156494 }, { "rung": "d1536", "full_compute": 1.3813258500033133e+21, "measured_compute": 1.3812345407853067e+21, "measured_progress": 0.9999338974087785, "measured_bpb": 0.8571977019309998, "measured_macro_loss": 2.5637738704681396 }, { "rung": "d2048", "full_compute": 7.741814947656449e+21, "measured_compute": 6.195534750109881e+21, "measured_progress": 0.8002690314866481, "measured_bpb": 0.829352855682373, "measured_macro_loss": 2.474504232406616, "forecast_full_bpb": 0.8081718863574893 } ], "nemotron_nano_measured_bpb": 0.7716271319301062, "nemotron_nano_training_compute": 5.25e+23, "nemotron_super_measured_bpb": 0.698188176710115, "nemotron_super_training_compute": 1.8e+24 }, "Gab": { "calibration": { "intercept": 0.40141377004129203, "slope": 0.43256513442616706, "r_squared": 0.998580478129264, "rmse": 0.0046711052838464525, "max_abs_residual": 0.01123723747717098, "n": 77 }, "hero_forecast_bpb": 1.2833209040328417, "sensitivity_forecasts": { "all_observations": 1.2833209040328417, "progress_ge_60_percent": 1.286805996241522, "last_observation_per_rung": 1.2928784154763002 }, "sensitivity_range": [ 1.2833209040328417, 1.2928784154763002 ], "conversion_bootstrap_95_percent": [ 1.278784476298934, 1.2932612544080437 ], "current_hero_measured_bpb": 1.4026343822479248, "current_hero_predicted_bpb": 1.3784688070834406, "qwen_measured_bpb": 1.4485291592424616, "qwen_training_compute": 6.48e+23, "endpoints": [ { "rung": "d768", "full_compute": 2.649679880270119e+19, "measured_compute": 2.6494478592648413e+19, "measured_progress": 0.9999124343257443, "measured_bpb": 1.7025781869888306, "measured_macro_loss": 3.015353202819824 }, { "rung": "d1024", "full_compute": 1.7261182741798453e+20, "measured_compute": 1.7260052787442483e+20, "measured_progress": 0.9999345378371302, "measured_bpb": 1.6065263748168945, "measured_macro_loss": 2.780874729156494 }, { "rung": "d1536", "full_compute": 1.3813258500033133e+21, "measured_compute": 1.3812345407853067e+21, "measured_progress": 0.9999338974087785, "measured_bpb": 1.5129204988479614, "measured_macro_loss": 2.5637738704681396 }, { "rung": "d2048", "full_compute": 7.741814947656449e+21, "measured_compute": 6.195534750109881e+21, "measured_progress": 0.8002690314866481, "measured_bpb": 1.4758743047714233, "measured_macro_loss": 2.474504232406616, "forecast_full_bpb": 1.4433369235622953 } ], "nemotron_nano_measured_bpb": 1.5994345990189138, "nemotron_nano_training_compute": 5.25e+23, "nemotron_super_measured_bpb": 1.35949649876937, "nemotron_super_training_compute": 1.8e+24 }, "M2D2 S2ORC": { "calibration": { "intercept": -0.10093267650732063, "slope": 0.3145347411551651, "r_squared": 0.9995152651043556, "rmse": 0.00198387673582053, "max_abs_residual": 0.005496640376022377, "n": 77 }, "hero_forecast_bpb": 0.5403359095078389, "sensitivity_forecasts": { "all_observations": 0.5403359095078389, "progress_ge_60_percent": 0.5395663930226505, "last_observation_per_rung": 0.5352160032330667 }, "sensitivity_range": [ 0.5352160032330667, 0.5403359095078389 ], "conversion_bootstrap_95_percent": [ 0.5380938671080074, 0.5450477002191032 ], "current_hero_measured_bpb": 0.6130969524383545, "current_hero_predicted_bpb": 0.6095216082053955, "qwen_measured_bpb": 0.6036988700101388, "qwen_training_compute": 6.48e+23, "endpoints": [ { "rung": "d768", "full_compute": 2.649679880270119e+19, "measured_compute": 2.6494478592648413e+19, "measured_progress": 0.9999124343257443, "measured_bpb": 0.8529973030090332, "measured_macro_loss": 3.015353202819824 }, { "rung": "d1024", "full_compute": 1.7261182741798453e+20, "measured_compute": 1.7260052787442483e+20, "measured_progress": 0.9999345378371302, "measured_bpb": 0.7775632739067078, "measured_macro_loss": 2.780874729156494 }, { "rung": "d1536", "full_compute": 1.3813258500033133e+21, "measured_compute": 1.3812345407853067e+21, "measured_progress": 0.9999338974087785, "measured_bpb": 0.708041250705719, "measured_macro_loss": 2.5637738704681396 }, { "rung": "d2048", "full_compute": 7.741814947656449e+21, "measured_compute": 6.195534750109881e+21, "measured_progress": 0.8002690314866481, "measured_bpb": 0.6756245493888855, "measured_macro_loss": 2.474504232406616, "forecast_full_bpb": 0.6566897098439297 } ], "nemotron_nano_measured_bpb": 0.614627039632233, "nemotron_nano_training_compute": 5.25e+23, "nemotron_super_measured_bpb": 0.573443445183165, "nemotron_super_training_compute": 1.8e+24 }, "M2D2 Wikipedia": { "calibration": { "intercept": -0.18584786123432864, "slope": 0.35407450040757654, "r_squared": 0.9996592993109145, "rmse": 0.001872163561161622, "max_abs_residual": 0.006319684312198204, "n": 77 }, "hero_forecast_bpb": 0.5360337769799371, "sensitivity_forecasts": { "all_observations": 0.5360337769799371, "progress_ge_60_percent": 0.5376488075297038, "last_observation_per_rung": 0.5373905808706348 }, "sensitivity_range": [ 0.5360337769799371, 0.5376488075297038 ], "conversion_bootstrap_95_percent": [ 0.5303710120193346, 0.5393438962694131 ], "current_hero_measured_bpb": 0.6118083000183105, "current_hero_predicted_bpb": 0.6139167212032647, "qwen_measured_bpb": 0.6249749651794375, "qwen_training_compute": 6.48e+23, "endpoints": [ { "rung": "d768", "full_compute": 2.649679880270119e+19, "measured_compute": 2.6494478592648413e+19, "measured_progress": 0.9999124343257443, "measured_bpb": 0.8797162771224976, "measured_macro_loss": 3.015353202819824 }, { "rung": "d1024", "full_compute": 1.7261182741798453e+20, "measured_compute": 1.7260052787442483e+20, "measured_progress": 0.9999345378371302, "measured_bpb": 0.796610951423645, "measured_macro_loss": 2.780874729156494 }, { "rung": "d1536", "full_compute": 1.3813258500033133e+21, "measured_compute": 1.3812345407853067e+21, "measured_progress": 0.9999338974087785, "measured_bpb": 0.7198629975318909, "measured_macro_loss": 2.5637738704681396 }, { "rung": "d2048", "full_compute": 7.741814947656449e+21, "measured_compute": 6.195534750109881e+21, "measured_progress": 0.8002690314866481, "measured_bpb": 0.6911993026733398, "measured_macro_loss": 2.474504232406616, "forecast_full_bpb": 0.6670142644555661 } ], "nemotron_nano_measured_bpb": 0.585591330234659, "nemotron_nano_training_compute": 5.25e+23, "nemotron_super_measured_bpb": 0.51002828168661, "nemotron_super_training_compute": 1.8e+24 }, "Manosphere": { "calibration": { "intercept": 0.1899379612203047, "slope": 0.3223080463073155, "r_squared": 0.9995038383285105, "rmse": 0.0020567388745889315, "max_abs_residual": 0.007838634903410524, "n": 77 }, "hero_forecast_bpb": 0.8470546420604641, "sensitivity_forecasts": { "all_observations": 0.8470546420604641, "progress_ge_60_percent": 0.842560702456031, "last_observation_per_rung": 0.8415648419897872 }, "sensitivity_range": [ 0.8415648419897872, 0.8470546420604641 ], "conversion_bootstrap_95_percent": [ 0.8398967127675186, 0.853902991487882 ], "current_hero_measured_bpb": 0.913962721824646, "current_hero_predicted_bpb": 0.9179501727027073, "qwen_measured_bpb": 0.9309705480715476, "qwen_training_compute": 6.48e+23, "endpoints": [ { "rung": "d768", "full_compute": 2.649679880270119e+19, "measured_compute": 2.6494478592648413e+19, "measured_progress": 0.9999124343257443, "measured_bpb": 1.1631572246551514, "measured_macro_loss": 3.015353202819824 }, { "rung": "d1024", "full_compute": 1.7261182741798453e+20, "measured_compute": 1.7260052787442483e+20, "measured_progress": 0.9999345378371302, "measured_bpb": 1.0870245695114136, "measured_macro_loss": 2.780874729156494 }, { "rung": "d1536", "full_compute": 1.3813258500033133e+21, "measured_compute": 1.3812345407853067e+21, "measured_progress": 0.9999338974087785, "measured_bpb": 1.014733076095581, "measured_macro_loss": 2.5637738704681396 }, { "rung": "d2048", "full_compute": 7.741814947656449e+21, "measured_compute": 6.195534750109881e+21, "measured_progress": 0.8002690314866481, "measured_bpb": 0.9848760962486267, "measured_macro_loss": 2.474504232406616, "forecast_full_bpb": 0.9662839708431206 } ], "nemotron_nano_measured_bpb": 0.9208411790061357, "nemotron_nano_training_compute": 5.25e+23, "nemotron_super_measured_bpb": 0.855564309099083, "nemotron_super_training_compute": 1.8e+24 }, "mC4": { "calibration": { "intercept": -0.08251803305917867, "slope": 0.3338989525684014, "r_squared": 0.9994453077568618, "rmse": 0.0022529430960078214, "max_abs_residual": 0.005057202309860953, "n": 77 }, "hero_forecast_bpb": 0.5982300089507602, "sensitivity_forecasts": { "all_observations": 0.5982300089507602, "progress_ge_60_percent": 0.5975343818375749, "last_observation_per_rung": 0.5954687936372559 }, "sensitivity_range": [ 0.5954687936372559, 0.5982300089507602 ], "conversion_bootstrap_95_percent": [ 0.5928180066425923, 0.5994837834815285 ], "current_hero_measured_bpb": 0.6633339524269104, "current_hero_predicted_bpb": 0.6716750988094701, "qwen_measured_bpb": 0.7142273244488536, "qwen_training_compute": 6.48e+23, "endpoints": [ { "rung": "d768", "full_compute": 2.649679880270119e+19, "measured_compute": 2.6494478592648413e+19, "measured_progress": 0.9999124343257443, "measured_bpb": 0.9205182194709778, "measured_macro_loss": 3.015353202819824 }, { "rung": "d1024", "full_compute": 1.7261182741798453e+20, "measured_compute": 1.7260052787442483e+20, "measured_progress": 0.9999345378371302, "measured_bpb": 0.8424513936042786, "measured_macro_loss": 2.780874729156494 }, { "rung": "d1536", "full_compute": 1.3813258500033133e+21, "measured_compute": 1.3812345407853067e+21, "measured_progress": 0.9999338974087785, "measured_bpb": 0.7684661746025085, "measured_macro_loss": 2.5637738704681396 }, { "rung": "d2048", "full_compute": 7.741814947656449e+21, "measured_compute": 6.195534750109881e+21, "measured_progress": 0.8002690314866481, "measured_bpb": 0.7417214512825012, "measured_macro_loss": 2.474504232406616, "forecast_full_bpb": 0.7217470866411007 } ], "nemotron_nano_measured_bpb": 0.7093700746669721, "nemotron_nano_training_compute": 5.25e+23, "nemotron_super_measured_bpb": 0.6271938873967, "nemotron_super_training_compute": 1.8e+24 }, "Penn Treebank": { "calibration": { "intercept": -0.32542821200119004, "slope": 0.45205619425523164, "r_squared": 0.997068434931368, "rmse": 0.007020504887518496, "max_abs_residual": 0.024100398622139307, "n": 77 }, "hero_forecast_bpb": 0.5962169945816957, "sensitivity_forecasts": { "all_observations": 0.5962169945816957, "progress_ge_60_percent": 0.5884214544426964, "last_observation_per_rung": 0.5831951293265231 }, "sensitivity_range": [ 0.5831951293265231, 0.5962169945816957 ], "conversion_bootstrap_95_percent": [ 0.5845353227563501, 0.6258587139002961 ], "current_hero_measured_bpb": 0.6807579398155212, "current_hero_predicted_bpb": 0.6956521906279021, "qwen_measured_bpb": 0.7139626869311755, "qwen_training_compute": 6.48e+23, "endpoints": [ { "rung": "d768", "full_compute": 2.649679880270119e+19, "measured_compute": 2.6494478592648413e+19, "measured_progress": 0.9999124343257443, "measured_bpb": 1.0533778667449951, "measured_macro_loss": 3.015353202819824 }, { "rung": "d1024", "full_compute": 1.7261182741798453e+20, "measured_compute": 1.7260052787442483e+20, "measured_progress": 0.9999345378371302, "measured_bpb": 0.9412280917167664, "measured_macro_loss": 2.780874729156494 }, { "rung": "d1536", "full_compute": 1.3813258500033133e+21, "measured_compute": 1.3812345407853067e+21, "measured_progress": 0.9999338974087785, "measured_bpb": 0.8395755887031555, "measured_macro_loss": 2.5637738704681396 }, { "rung": "d2048", "full_compute": 7.741814947656449e+21, "measured_compute": 6.195534750109881e+21, "measured_progress": 0.8002690314866481, "measured_bpb": 0.7904568314552307, "measured_macro_loss": 2.474504232406616, "forecast_full_bpb": 0.7634432156785846 } ], "nemotron_nano_measured_bpb": 0.9696189369488944, "nemotron_nano_training_compute": 5.25e+23, "nemotron_super_measured_bpb": 0.99854091745094, "nemotron_super_training_compute": 1.8e+24 }, "RedPajama": { "calibration": { "intercept": -0.19816308294069412, "slope": 0.33649348384994215, "r_squared": 0.9988712739671078, "rmse": 0.0032397000595864125, "max_abs_residual": 0.013031017190242133, "n": 77 }, "hero_forecast_bpb": 0.48787464948682874, "sensitivity_forecasts": { "all_observations": 0.48787464948682874, "progress_ge_60_percent": 0.49411100551584086, "last_observation_per_rung": 0.4968082560292394 }, "sensitivity_range": [ 0.48787464948682874, 0.4968082560292394 ], "conversion_bootstrap_95_percent": [ 0.4725442695035298, 0.49656324050703693 ], "current_hero_measured_bpb": 0.5743716359138489, "current_hero_predicted_bpb": 0.5618904377009788, "qwen_measured_bpb": 0.5494597468475731, "qwen_training_compute": 6.48e+23, "endpoints": [ { "rung": "d768", "full_compute": 2.649679880270119e+19, "measured_compute": 2.6494478592648413e+19, "measured_progress": 0.9999124343257443, "measured_bpb": 0.8125020861625671, "measured_macro_loss": 3.015353202819824 }, { "rung": "d1024", "full_compute": 1.7261182741798453e+20, "measured_compute": 1.7260052787442483e+20, "measured_progress": 0.9999345378371302, "measured_bpb": 0.7337654829025269, "measured_macro_loss": 2.780874729156494 }, { "rung": "d1536", "full_compute": 1.3813258500033133e+21, "measured_compute": 1.3812345407853067e+21, "measured_progress": 0.9999338974087785, "measured_bpb": 0.6636402606964111, "measured_macro_loss": 2.5637738704681396 }, { "rung": "d2048", "full_compute": 7.741814947656449e+21, "measured_compute": 6.195534750109881e+21, "measured_progress": 0.8002690314866481, "measured_bpb": 0.639616847038269, "measured_macro_loss": 2.474504232406616, "forecast_full_bpb": 0.6123515053358217 } ], "nemotron_nano_measured_bpb": 0.5666622692956341, "nemotron_nano_training_compute": 5.25e+23, "nemotron_super_measured_bpb": 0.502868783415523, "nemotron_super_training_compute": 1.8e+24 }, "Twitter AAE": { "calibration": { "intercept": 1.3339004445321634, "slope": 0.38632294796868155, "r_squared": 0.9706146448334734, "rmse": 0.0192522368050697, "max_abs_residual": 0.0562289829652558, "n": 77 }, "hero_forecast_bpb": 2.121529720965166, "sensitivity_forecasts": { "all_observations": 2.121529720965166, "progress_ge_60_percent": 2.1134966434507456, "last_observation_per_rung": 2.132121011594017 }, "sensitivity_range": [ 2.1134966434507456, 2.132121011594017 ], "conversion_bootstrap_95_percent": [ 2.1075042939482964, 2.1759647153489556 ], "current_hero_measured_bpb": 2.196295976638794, "current_hero_predicted_bpb": 2.2065060990274152, "qwen_measured_bpb": 2.270256057355065, "qwen_training_compute": 6.48e+23, "endpoints": [ { "rung": "d768", "full_compute": 2.649679880270119e+19, "measured_compute": 2.6494478592648413e+19, "measured_progress": 0.9999124343257443, "measured_bpb": 2.505134105682373, "measured_macro_loss": 3.015353202819824 }, { "rung": "d1024", "full_compute": 1.7261182741798453e+20, "measured_compute": 1.7260052787442483e+20, "measured_progress": 0.9999345378371302, "measured_bpb": 2.435474157333374, "measured_macro_loss": 2.780874729156494 }, { "rung": "d1536", "full_compute": 1.3813258500033133e+21, "measured_compute": 1.3812345407853067e+21, "measured_progress": 0.9999338974087785, "measured_bpb": 2.3402576446533203, "measured_macro_loss": 2.5637738704681396 }, { "rung": "d2048", "full_compute": 7.741814947656449e+21, "measured_compute": 6.195534750109881e+21, "measured_progress": 0.8002690314866481, "measured_bpb": 2.2936666011810303, "measured_macro_loss": 2.474504232406616, "forecast_full_bpb": 2.2644396678540644 } ], "nemotron_nano_measured_bpb": 2.594239704754665, "nemotron_nano_training_compute": 5.25e+23, "nemotron_super_measured_bpb": 2.2442665067808, "nemotron_super_training_compute": 1.8e+24 }, "WikiText-103": { "calibration": { "intercept": -0.3449075894491445, "slope": 0.4037972557080258, "r_squared": 0.9994501557395197, "rmse": 0.0027126344776835, "max_abs_residual": 0.008881796008572973, "n": 77 }, "hero_forecast_bpb": 0.4783480364753285, "sensitivity_forecasts": { "all_observations": 0.4783480364753285, "progress_ge_60_percent": 0.4780425906480077, "last_observation_per_rung": 0.47464624827962976 }, "sensitivity_range": [ 0.47464624827962976, 0.4783480364753285 ], "conversion_bootstrap_95_percent": [ 0.47702582400246046, 0.4830105308028816 ], "current_hero_measured_bpb": 0.5587969422340393, "current_hero_predicted_bpb": 0.5671680986589647, "qwen_measured_bpb": 0.5465357490670965, "qwen_training_compute": 6.48e+23, "endpoints": [ { "rung": "d768", "full_compute": 2.649679880270119e+19, "measured_compute": 2.6494478592648413e+19, "measured_progress": 0.9999124343257443, "measured_bpb": 0.8757628202438354, "measured_macro_loss": 3.015353202819824 }, { "rung": "d1024", "full_compute": 1.7261182741798453e+20, "measured_compute": 1.7260052787442483e+20, "measured_progress": 0.9999345378371302, "measured_bpb": 0.7800092101097107, "measured_macro_loss": 2.780874729156494 }, { "rung": "d1536", "full_compute": 1.3813258500033133e+21, "measured_compute": 1.3812345407853067e+21, "measured_progress": 0.9999338974087785, "measured_bpb": 0.6902027726173401, "measured_macro_loss": 2.5637738704681396 }, { "rung": "d2048", "full_compute": 7.741814947656449e+21, "measured_compute": 6.195534750109881e+21, "measured_progress": 0.8002690314866481, "measured_bpb": 0.6536816358566284, "measured_macro_loss": 2.474504232406616, "forecast_full_bpb": 0.6277221408886002 } ], "nemotron_nano_measured_bpb": 0.5466383599906303, "nemotron_nano_training_compute": 5.25e+23, "nemotron_super_measured_bpb": 0.438013861770241, "nemotron_super_training_compute": 1.8e+24 } } }