{ "generated_at": "2026-09-01", "qwen": { "model": "Qwen/Qwen3.5-35B-A3B-Base", "revision": "0f0813072d2358973511097385626f21fcb6d422", "paloma_macro_bpb": 0.839683049996478, "uncheatable_macro_bpb": 0.5244890328654392, "paloma": { "4chan": 0.88278930596034, "C4 100 domains": 0.7045068602207951, "C4 English": 0.7549489519312605, "Dolma v1.5": 0.6582074468864063, "100 programming languages": 0.33941854758748374, "100 subreddits": 0.9136682068914669, "Falcon RefinedWeb": 0.7787743733125458, "Gab": 1.4485291592424616, "M2D2 S2ORC": 0.6036988700101388, "M2D2 Wikipedia": 0.6249749651794375, "Manosphere": 0.9309705480715476, "mC4": 0.7142273244488536, "Penn Treebank": 0.7139626869311755, "RedPajama": 0.5494597468475731, "Twitter AAE": 2.270256057355065, "WikiText-103": 0.5465357490670965 }, "uncheatable": { "Wikipedia English": 0.6175520035935521, "GitHub Python": 0.24789314432218026, "GitHub C++": 0.2464697715067429, "BBC News": 0.6667409852719914, "arXiv Physics": 0.5461191385120038, "arXiv Computer Science": 0.5385839797687898, "AO3 English": 0.8080642070828142 }, "training_tokens": 36000000000000, "active_parameters": 3000000000, "estimated_training_flops": 6.48e+23, "training_flops_method": "6 \u00d7 active parameters \u00d7 training tokens", "training_flops_caveat": "The 36T-token count is an input assumption. The model card reports 3B active parameters. The estimate uses the standard 6ND approximation; exact hybrid-MoE compute may differ." }, "hero": { "wandb_run": "https://wandb.ai/marin-community/marin_moe/runs/hero-12d8b6f0-dee637", "step": 45837, "paloma_macro_bpb_dropless": 0.8187980651855469, "uncheatable_macro_bpb_dropless": 0.5361782908439636, "paloma_macro_bpb_with_dropping": 0.8867964744567871, "uncheatable_macro_bpb_with_dropping": 0.6219991445541382, "paloma_dropless": { "4chan": 0.8472199440002441, "C4 100 domains": 0.6743637919425964, "C4 English": 0.7177862524986267, "Dolma v1.5": 0.650954008102417, "100 programming languages": 0.3567442297935486, "100 subreddits": 0.8852283954620361, "Falcon RefinedWeb": 0.7534131407737732, "Gab": 1.4026343822479248, "M2D2 S2ORC": 0.6130969524383545, "M2D2 Wikipedia": 0.6118083000183105, "Manosphere": 0.913962721824646, "mC4": 0.6633339524269104, "Penn Treebank": 0.6807579398155212, "RedPajama": 0.5743716359138489, "Twitter AAE": 2.196295976638794, "WikiText-103": 0.5587969422340393 }, "uncheatable_dropless": { "Wikipedia English": 0.6161309480667114, "GitHub Python": 0.2710290849208832, "GitHub C++": 0.2756171226501465, "BBC News": 0.6591649055480957, "arXiv Physics": 0.5689177513122559, "arXiv Computer Science": 0.5548429489135742, "AO3 English": 0.80754554271698 } }, "datasets": { "paloma_revision": "65cd6fc59dba021b21db414fa5e8d7765ffbe5e6", "uncheatable_revision": "185c463882a8ae0f203e51ae5852d8cf4fe299cf", "paloma_documents": 154440, "uncheatable_documents": 7052, "uncheatable_categories": 7 }, "nemotron_nano": { "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", "revision": "e0ac9ee3dfd02be21b5479edfc2f671ed269d0a2", "model_card": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16", "evaldash_run": "https://evaldash.oa.dev/runs/20260901-040435-nemotron-3-nano-30b-a3b-base-paloma-uncheatable-c989", "training_tokens": 25000000000000, "active_parameters": 3500000000, "estimated_training_flops": 5.25e+23, "training_flops_method": "6 \u00d7 active parameters \u00d7 training tokens", "training_flops_caveat": "NVIDIA publishes 25T training tokens and 3.5B active parameters, not an exact training-FLOP total. The estimate uses the standard 6ND approximation; exact hybrid-MoE compute may differ.", "paloma_macro_bpb": 0.8845696062703191, "uncheatable_macro_bpb": 0.5702917406634672, "paloma": { "4chan": 0.8529030813768576, "C4 100 domains": 0.7004211698724362, "C4 English": 0.747164483671727, "Dolma v1.5": 0.6621254426290294, "100 programming languages": 0.39262551987799477, "100 subreddits": 0.9192233774182177, "Falcon RefinedWeb": 0.7716271319301062, "Gab": 1.5994345990189138, "M2D2 S2ORC": 0.614627039632233, "M2D2 Wikipedia": 0.585591330234659, "Manosphere": 0.9208411790061357, "mC4": 0.7093700746669721, "Penn Treebank": 0.9696189369488944, "RedPajama": 0.5666622692956341, "Twitter AAE": 2.594239704754665, "WikiText-103": 0.5466383599906303 }, "uncheatable": { "AO3 English": 0.7893776639349719, "AO3 non-English": 1.0515490545396535, "arXiv Computer Science": 0.5618662527134773, "arXiv Math": 0.512925173811512, "arXiv Other": 0.5288103776095244, "arXiv Physics": 0.5353711646983408, "BBC News": 0.6579406207407943, "bioRxiv": 0.5417999602966902, "GitHub C++": 0.3012240622455594, "GitHub JavaScript": 0.3605569606623726, "GitHub Markdown": 0.704002499592267, "GitHub Other": 0.38411073966759013, "GitHub Python": 0.3843868020835418, "Wikipedia English": 0.6103702494803906, "Wikipedia non-English": 0.6300845278753212 }, "datasets": { "paloma_revision": "65cd6fc59dba021b21db414fa5e8d7765ffbe5e6", "paloma_documents": 154440, "uncheatable_revision": "65889535d56aa38d448ce7e07b08e6e36c031545", "uncheatable_documents": 7500, "uncheatable_categories": 15 }, "evalchemy_revision": "ea1cb9a759fca76a6535beeb05f5faca67c1b0fb" }, "nemotron_super": { "model": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16", "revision": "b4cc202094ea3644bc4461c9fe58ec8b6a3695eb", "model_card": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16", "evaldash_run": "https://evaldash.oa.dev/runs/20260901-045340-nemotron-3-super-120b-a12b-base-paloma-uncheatable-9329", "training_tokens": 25000000000000, "active_parameters": 12000000000, "estimated_training_flops": 1.8e+24, "training_flops_method": "6 \u00d7 active parameters \u00d7 training tokens", "training_flops_caveat": "NVIDIA reports over 25T training tokens and 12B active parameters. The 1.8e24-FLOP coordinate uses 25T, so it is a lower-bound 6ND estimate; exact hybrid-MoE compute may differ.", "paloma_macro_bpb": 0.7917556908564041, "uncheatable_macro_bpb": 0.496361297843761, "paloma": { "4chan": 0.793467863723923, "C4 100 domains": 0.633082710860957, "C4 English": 0.675213142735745, "Dolma v1.5": 0.593682537587207, "100 programming languages": 0.323534044276444, "100 subreddits": 0.841506086255642, "Falcon RefinedWeb": 0.698188176710115, "Gab": 1.35949649876937, "M2D2 S2ORC": 0.573443445183165, "M2D2 Wikipedia": 0.51002828168661, "Manosphere": 0.855564309099083, "mC4": 0.6271938873967, "Penn Treebank": 0.99854091745094, "RedPajama": 0.502868783415523, "Twitter AAE": 2.2442665067808, "WikiText-103": 0.438013861770241 }, "uncheatable": { "AO3 English": 0.733737084357567, "AO3 non-English": 0.94004935604982, "arXiv Computer Science": 0.499634721203136, "arXiv Math": 0.447317378753065, "arXiv Other": 0.46900424811776, "arXiv Physics": 0.478924657071078, "BBC News": 0.611606791555797, "bioRxiv": 0.492478539762714, "GitHub C++": 0.239828237605929, "GitHub JavaScript": 0.279234635088224, "GitHub Markdown": 0.589091053759629, "GitHub Other": 0.301990872885754, "GitHub Python": 0.296254281908053, "Wikipedia English": 0.54631616467185, "Wikipedia non-English": 0.519951444866047 }, "datasets": { "paloma_revision": "65cd6fc59dba021b21db414fa5e8d7765ffbe5e6", "paloma_documents": 154440, "uncheatable_revision": "65889535d56aa38d448ce7e07b08e6e36c031545", "uncheatable_documents": 7500, "uncheatable_categories": 15 }, "evalchemy_revision": "ea1cb9a759fca76a6535beeb05f5faca67c1b0fb" } }