{
  "metrics": [
    {
      "name": "num_perplexity_tokens",
      "display_name": "# tokens",
      "description": "Average number of tokens in the predicted output (for language modeling, the input too)."
    },
    {
      "name": "num_bytes",
      "display_name": "# bytes",
      "description": "Average number of bytes in the predicted output (for language modeling, the input too)."
    },
    {
      "name": "num_references",
      "display_name": "# ref",
      "description": "Number of references."
    },
    {
      "name": "num_train_trials",
      "display_name": "# trials",
      "description": "Number of trials, where in each trial we choose an independent, random set of training instances."
    },
    {
      "name": "estimated_num_tokens_cost",
      "display_name": "cost",
      "description": "An estimate of the number of tokens (including prompt and output completions) needed to perform the request."
    },
    {
      "name": "num_prompt_tokens",
      "display_name": "# prompt tokens",
      "description": "Number of tokens in the prompt."
    },
    {
      "name": "num_prompt_characters",
      "display_name": "# prompt chars",
      "description": "Number of characters in the prompt."
    },
    {
      "name": "num_completion_tokens",
      "display_name": "# completion tokens",
      "description": "Actual number of completion tokens (over all completions)."
    },
    {
      "name": "num_output_tokens",
      "display_name": "# output tokens",
      "description": "Actual number of output tokens."
    },
    {
      "name": "max_num_output_tokens",
      "display_name": "Max output tokens",
      "description": "Maximum number of output tokens (overestimate since we might stop earlier due to stop sequences)."
    },
    {
      "name": "num_requests",
      "display_name": "# requests",
      "description": "Number of distinct API requests."
    },
    {
      "name": "num_instances",
      "display_name": "# eval",
      "description": "Number of evaluation instances."
    },
    {
      "name": "num_train_instances",
      "display_name": "# train",
      "description": "Number of training instances (e.g., in-context examples)."
    },
    {
      "name": "prompt_truncated",
      "display_name": "truncated",
      "description": "Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples)."
    },
    {
      "name": "finish_reason_length",
      "display_name": "finish b/c length",
      "description": "Fraction of instances where the the output was terminated because of the max tokens limit."
    },
    {
      "name": "finish_reason_stop",
      "display_name": "finish b/c stop",
      "description": "Fraction of instances where the the output was terminated because of the stop sequences."
    },
    {
      "name": "finish_reason_endoftext",
      "display_name": "finish b/c endoftext",
      "description": "Fraction of instances where the the output was terminated because the end of text token was generated."
    },
    {
      "name": "finish_reason_unknown",
      "display_name": "finish b/c unknown",
      "description": "Fraction of instances where the the output was terminated for unknown reasons."
    },
    {
      "name": "num_completions",
      "display_name": "# completions",
      "description": "Number of completions."
    },
    {
      "name": "predicted_index",
      "display_name": "Predicted index",
      "description": "Integer index of the reference (0, 1, ...) that was predicted by the model (for multiple-choice)."
    },
    {
      "name": "inference_runtime",
      "display_name": "Observed inference runtime (s)",
      "short_display_name": "Observed inference time (s)",
      "description": "Average observed time to process a request to the model (via an API, and thus depends on particular deployment).",
      "lower_is_better": true
    },
    {
      "name": "exact_match",
      "display_name": "Exact match",
      "short_display_name": "EM",
      "description": "Fraction of instances that the predicted output matches a correct reference exactly.",
      "lower_is_better": false
    },
    {
      "name": "quasi_exact_match",
      "display_name": "Quasi-exact match",
      "short_display_name": "EM",
      "description": "Fraction of instances that the predicted output matches a correct reference up to light processing.",
      "lower_is_better": false
    },
    {
      "name": "prefix_exact_match",
      "display_name": "Prefix exact match",
      "short_display_name": "PEM",
      "description": "Fraction of instances that the predicted output matches the prefix of a correct reference exactly.",
      "lower_is_better": false
    },
    {
      "name": "quasi_prefix_exact_match",
      "display_name": "Prefix quasi-exact match",
      "short_display_name": "PEM",
      "description": "Fraction of instances that the predicted output matches the prefix of a correct reference up to light processing.",
      "lower_is_better": false
    },
    {
      "name": "alrage_score",
      "display_name": "ALRAGE Score",
      "short_display_name": "Score",
      "description": "Score of the output judged by GPT-4o.",
      "lower_is_better": false
    }
  ],
  "perturbations": [],
  "metric_groups": [
    {
      "name": "accuracy",
      "display_name": "Accuracy",
      "metrics": [
        {
          "name": "${main_name}",
          "split": "${main_split}"
        }
      ],
      "aggregation_strategies": [
        "mean"
      ]
    },
    {
      "name": "efficiency",
      "display_name": "Efficiency",
      "metrics": [
        {
          "name": "inference_runtime",
          "split": "${main_split}"
        }
      ],
      "aggregation_strategies": [
        "mean"
      ]
    },
    {
      "name": "general_information",
      "display_name": "General information",
      "metrics": [
        {
          "name": "num_instances",
          "split": "${main_split}"
        },
        {
          "name": "num_train_instances",
          "split": "${main_split}"
        },
        {
          "name": "prompt_truncated",
          "split": "${main_split}"
        },
        {
          "name": "num_prompt_tokens",
          "split": "${main_split}"
        },
        {
          "name": "num_output_tokens",
          "split": "${main_split}"
        }
      ],
      "hide_win_rates": true
    }
  ],
  "run_groups": [
    {
      "name": "arabic_scenarios",
      "display_name": "Arabic Scenarios",
      "description": "Arabic Scenarios",
      "metric_groups": [],
      "subgroups": [
        "alghafa",
        "arabic_mmlu",
        "arabic_exams",
        "madinah_qa",
        "aratrust",
        "alrage",
        "mbzuai_human_translated_arabic_mmlu"
      ],
      "subgroup_display_mode": "by_metric",
      "subgroup_metric_groups_hidden": [],
      "environment": {},
      "category": "Scenarios",
      "visibility": "all_groups",
      "todo": false,
      "adapter_keys_shown": [
        "model_deployment",
        "model"
      ]
    },
    {
      "name": "alghafa",
      "display_name": "AlGhafa",
      "description": "An Arabic language multiple choice evaluation benchmark derived from publicly available NLP datasets ([paper](https://aclanthology.org/2023.arabicnlp-1.21/))",
      "metric_groups": [
        "accuracy",
        "efficiency",
        "general_information"
      ],
      "subgroups": [],
      "subgroup_display_mode": "by_metric",
      "subgroup_metric_groups_hidden": [],
      "environment": {
        "main_name": "exact_match",
        "main_split": "test"
      },
      "category": "Scenarios",
      "visibility": "all_groups",
      "taxonomy": {
        "task": "multiple choice question answering",
        "what": "Various",
        "when": "before 2023",
        "who": "Various",
        "language": "Arabic"
      },
      "todo": false,
      "adapter_keys_shown": [
        "model_deployment",
        "model"
      ]
    },
    {
      "name": "arabic_mmlu",
      "display_name": "ArabicMMLU",
      "description": "A native Arabic language question answering benchmark using questions sourced from school exams across diverse educational levels in different countries spanning North Africa, the Levant, and the Gulf regions ([dataset](https://huggingface.co/datasets/MBZUAI/ArabicMMLU))",
      "metric_groups": [
        "accuracy",
        "efficiency",
        "general_information"
      ],
      "subgroups": [],
      "subgroup_display_mode": "by_metric",
      "subgroup_metric_groups_hidden": [],
      "environment": {
        "main_name": "exact_match",
        "main_split": "test"
      },
      "category": "Scenarios",
      "visibility": "all_groups",
      "taxonomy": {
        "task": "question answering",
        "what": "academic questions across various disciplines",
        "when": "before 2024",
        "who": "academic exams writers and takers",
        "language": "Arabic"
      },
      "todo": false,
      "adapter_keys_shown": [
        "model_deployment",
        "model"
      ]
    },
    {
      "name": "arabic_exams",
      "display_name": "Arabic EXAMS",
      "description": "The Arabic language subset of the EXAMS multilingual question answering benchmark, which consists of high school exam questions across various school subjects ([paper](https://aclanthology.org/2020.emnlp-main.438/))",
      "metric_groups": [
        "accuracy",
        "efficiency",
        "general_information"
      ],
      "subgroups": [],
      "subgroup_display_mode": "by_metric",
      "subgroup_metric_groups_hidden": [],
      "environment": {
        "main_name": "exact_match",
        "main_split": "test"
      },
      "category": "Scenarios",
      "visibility": "all_groups",
      "taxonomy": {
        "task": "multiple choice question answering",
        "what": "High school examinations",
        "when": "before 2020",
        "who": "High school examinations writers and test-takers",
        "language": "Arabic"
      },
      "todo": false,
      "adapter_keys_shown": [
        "model_deployment",
        "model"
      ]
    },
    {
      "name": "madinah_qa",
      "display_name": "MadinahQA",
      "description": "A question answering benchmark published by MBZUAI that tests knowledge of Arabic language and grammar ([dataset](https://huggingface.co/datasets/MBZUAI/MadinahQA))",
      "metric_groups": [
        "accuracy",
        "efficiency",
        "general_information"
      ],
      "subgroups": [],
      "subgroup_display_mode": "by_metric",
      "subgroup_metric_groups_hidden": [],
      "environment": {
        "main_name": "exact_match",
        "main_split": "test"
      },
      "category": "Scenarios",
      "visibility": "all_groups",
      "taxonomy": {
        "task": "question answering",
        "what": "academic questions about Arabic language",
        "when": "before 2024",
        "who": "academic exams writers and takers",
        "language": "Arabic"
      },
      "todo": false,
      "adapter_keys_shown": [
        "model_deployment",
        "model"
      ]
    },
    {
      "name": "aratrust",
      "display_name": "AraTrust",
      "description": "An Arab-region-specific safety evaluation dataset consisting of human-written questions including direct attacks, indirect attacks, and harmless requests with sensitive words ([paper](https://arxiv.org/abs/2410.17040))",
      "metric_groups": [
        "accuracy",
        "efficiency",
        "general_information"
      ],
      "subgroups": [],
      "subgroup_display_mode": "by_metric",
      "subgroup_metric_groups_hidden": [],
      "environment": {
        "main_name": "exact_match",
        "main_split": "test"
      },
      "category": "Scenarios",
      "visibility": "all_groups",
      "taxonomy": {
        "task": "safety",
        "what": "human-written questions including direct attacks, indirect attacks, and harmless requests",
        "when": "2024",
        "who": "human question writers",
        "language": "Arabic"
      },
      "todo": false,
      "adapter_keys_shown": [
        "model_deployment",
        "model"
      ]
    },
    {
      "name": "alrage",
      "display_name": "ALRAGE",
      "description": "An Arabic language passage-based open-ended model-graded question answering benchmark that reflects retrieval-augmented generation use cases ([dataset](https://huggingface.co/datasets/OALL/ALRAGE))",
      "metric_groups": [
        "accuracy",
        "efficiency",
        "general_information"
      ],
      "subgroups": [],
      "subgroup_display_mode": "by_metric",
      "subgroup_metric_groups_hidden": [],
      "environment": {
        "main_name": "alrage_score",
        "main_split": "test"
      },
      "category": "Scenarios",
      "visibility": "all_groups",
      "taxonomy": {
        "task": "openbook (RAG) open-ended question answering",
        "what": "passage-based question answering",
        "when": "before 2025",
        "who": "book authors",
        "language": "Arabic"
      },
      "todo": false,
      "adapter_keys_shown": [
        "model_deployment",
        "model"
      ]
    },
    {
      "name": "mbzuai_human_translated_arabic_mmlu",
      "display_name": "MBZUAI Human-Translated Arabic MMLU",
      "short_display_name": "Translated MMLU",
      "description": "A translation of MMLU to Arabic by human translators published by MBZUAI ([dataset](https://huggingface.co/datasets/MBZUAI/human_translated_arabic_mmlu))",
      "metric_groups": [
        "accuracy",
        "efficiency",
        "general_information"
      ],
      "subgroups": [],
      "subgroup_display_mode": "by_metric",
      "subgroup_metric_groups_hidden": [],
      "environment": {
        "main_name": "exact_match",
        "main_split": "test"
      },
      "category": "Scenarios",
      "visibility": "all_groups",
      "taxonomy": {
        "task": "multiple-choice question answering",
        "what": "math, science, history, etc.",
        "when": "before 2021",
        "who": "various online sources",
        "language": "Arabic"
      },
      "todo": false,
      "adapter_keys_shown": [
        "model_deployment",
        "model"
      ]
    }
  ],
  "adapter": [
    {
      "name": "method",
      "description": "The high-level strategy for converting instances into a prompt for the language model."
    },
    {
      "name": "global_prefix",
      "description": "The string that is prepended to the entire prompt."
    },
    {
      "name": "global_suffix",
      "description": "The string that is appended to the entire prompt."
    },
    {
      "name": "instructions",
      "description": "The description of the task that is included at the very beginning of the prompt."
    },
    {
      "name": "input_prefix",
      "description": "The string that is included before each input (e.g., 'Question:')."
    },
    {
      "name": "input_suffix",
      "description": "The string that is included after each input (e.g., '\\n')."
    },
    {
      "name": "reference_prefix",
      "description": "The string that is included before each reference (for multiple-choice questions)."
    },
    {
      "name": "reference_prefix_characters",
      "description": "The characters that are used to identify choices for multiple-choice questions e.g. [\"A\", \"B\", \"C\", \"D\"]. If unset, defaults to the sequence of ascending characters starting from the first character of reference_prefix."
    },
    {
      "name": "reference_suffix",
      "description": "The string that is included after each reference (for multiple-choice questions)."
    },
    {
      "name": "chain_of_thought_prefix",
      "description": "The string that is included before each chain of thought. (e.g., 'Let's think step by step')"
    },
    {
      "name": "chain_of_thought_suffix",
      "description": "The string that is included after each chain of thought. (e.g., 'The correct answer is')"
    },
    {
      "name": "output_prefix",
      "description": "The string that is included before the correct answer/predicted output (e.g., 'Answer:')."
    },
    {
      "name": "output_suffix",
      "description": "The string that is included after the correct answer/predicted output (e.g., '\\n')."
    },
    {
      "name": "instance_prefix",
      "description": "The string that is included before each instance (e.g., '\\n\\n')."
    },
    {
      "name": "substitutions",
      "description": "A list of regular expression substitutions (e.g., replacing '\\n' with ';\\n') to perform at the very end on the prompt."
    },
    {
      "name": "max_train_instances",
      "description": "Maximum number of training instances to include in the prompt (currently by randomly sampling)."
    },
    {
      "name": "max_eval_instances",
      "description": "Maximum number of instances to evaluate on (over all splits - test, valid, etc.)."
    },
    {
      "name": "num_outputs",
      "description": "Maximum number of possible outputs to generate by sampling multiple outputs."
    },
    {
      "name": "num_train_trials",
      "description": "Number of trials, where in each trial we choose an independent, random set of training instances. Used to compute variance."
    },
    {
      "name": "num_trials",
      "description": "Number of trials, where we query the model with the same requests, but different random seeds."
    },
    {
      "name": "sample_train",
      "description": "If true, randomly sample N training examples; if false, select N consecutive training examples"
    },
    {
      "name": "model_deployment",
      "description": "Name of the language model deployment (<host_organization>/<model name>) to send requests to."
    },
    {
      "name": "model",
      "description": "Name of the language model (<creator_organization>/<model name>) to send requests to."
    },
    {
      "name": "temperature",
      "description": "Temperature parameter used in generation."
    },
    {
      "name": "max_tokens",
      "description": "Maximum number of tokens to generate."
    },
    {
      "name": "stop_sequences",
      "description": "List of stop sequences. Output generation will be stopped if any stop sequence is encountered."
    },
    {
      "name": "random",
      "description": "Random seed (string), which guarantees reproducibility."
    },
    {
      "name": "multi_label",
      "description": "If true, for instances with multiple correct reference, the gold answer should be considered to be all of the correct references rather than any of the correct references."
    },
    {
      "name": "image_generation_parameters",
      "description": "Parameters for image generation."
    },
    {
      "name": "reeval_parameters",
      "description": "Parameters for reeval evaluation."
    },
    {
      "name": "eval_splits",
      "description": "The splits from which evaluation instances will be drawn."
    },
    {
      "name": "output_mapping_pattern",
      "description": "Pattern to apply to the output before applying the output mapping for the joint multiple choice adapter. If the pattern has no group, the output mapping will be applied to the first match. If the pattern has a group, the output mapping will be applied to the group of the first match."
    }
  ],
  "models": [
    {
      "name": "arabicai/llm-x",
      "display_name": "Arabic.AI LLM-X (Preview)",
      "short_display_name": "Arabic.AI LLM-X (Preview)",
      "description": "Arabic.AI (Preview) LLM-X is a flagship model, designed with Arabic at its core and optimized to deliver enterprise-grade performance, strong agentic capabilities, and scalability for complex workloads.",
      "creator_organization": "Arabic.AI",
      "access": "limited",
      "todo": false
    },
    {
      "name": "arabicai/llm-s",
      "display_name": "Arabic.AI LLM-S",
      "short_display_name": "Arabic.AI LLM-S",
      "description": "Arabic.AI LLM-S is a compact model, designed with Arabic at its core and optimized to deliver efficient enterprise-grade performance, faster inference, and scalability for production workloads.",
      "creator_organization": "Arabic.AI",
      "access": "limited",
      "todo": false
    },
    {
      "name": "anthropic/claude-haiku-4-5-20251001",
      "display_name": "Claude 4.5 Haiku (20251001)",
      "short_display_name": "Claude 4.5 Haiku (20251001)",
      "description": "Claude 4.5 Haiku is a hybrid model from Anthropic in their small, fast model class that is particularly effective at coding tasks and computer use. ([blog](https://www.anthropic.com/news/claude-haiku-4-5), [system card](https://assets.anthropic.com/m/99128ddd009bdcb/Claude-Haiku-4-5-System-Card.pdf))",
      "creator_organization": "Anthropic",
      "access": "limited",
      "todo": false,
      "release_date": "2025-10-15"
    },
    {
      "name": "anthropic/claude-sonnet-4-6",
      "display_name": "Claude 4.6 Sonnet",
      "short_display_name": "Claude 4.6 Sonnet",
      "description": "Claude 4.6 Sonnet is a Sonnet model from Anthropic that upgrades Sonnet's skills across coding, computer use, long-context reasoning, agent planning, knowledge work, and design. ([blog](https://www.anthropic.com/news/claude-sonnet-4-6), [system card](https://www-cdn.anthropic.com/78073f739564e986ff3e28522761a7a0b4484f84.pdf))",
      "creator_organization": "Anthropic",
      "access": "limited",
      "todo": false,
      "release_date": "2026-02-17"
    },
    {
      "name": "anthropic/claude-opus-4-7",
      "display_name": "Claude 4.7 Opus",
      "short_display_name": "Claude 4.7 Opus",
      "description": "Claude 4.7 Opus is a large language model with particular skills in areas such as software engineering, knowledge work, agentic tool use, and computer use. ([blog](https://www.anthropic.com/news/claude-opus-4-7), [system card](https://anthropic.com/claude-opus-4-7-system-card))",
      "creator_organization": "Anthropic",
      "access": "limited",
      "todo": false,
      "release_date": "2026-04-16"
    },
    {
      "name": "cohere/command-a-03-2025",
      "display_name": "Cohere Labs Command A",
      "short_display_name": "Cohere Labs Command A",
      "description": "Cohere Labs Command A is an open weights research release of a 111 billion parameter model optimized for enterprise use-cases. ([blog](https://cohere.com/blog/command-a), [paper](https://arxiv.org/abs/2504.00698))",
      "creator_organization": "Cohere",
      "access": "open",
      "todo": false,
      "release_date": "2025-03-13",
      "num_parameters": 111000000000
    },
    {
      "name": "deepseek-ai/deepseek-v3.1",
      "display_name": "DeepSeek v3.1",
      "short_display_name": "DeepSeek v3.1",
      "description": "DeepSeek v3.1 is a hybrid model that supports both thinking mode and non-thinking mode. ([blog](https://api-docs.deepseek.com/news/news250821))",
      "creator_organization": "DeepSeek",
      "access": "open",
      "todo": false,
      "release_date": "2025-08-21",
      "num_parameters": 685000000000
    },
    {
      "name": "google/gemini-2.5-flash-lite-thinking-disabled",
      "display_name": "Gemini 2.5 Flash-Lite (thinking disabled)",
      "short_display_name": "Gemini 2.5 Flash-Lite (thinking disabled)",
      "description": "Gemini 2.5 Flash-Lite with thinking disabled ([blog](https://blog.google/products/gemini/gemini-2-5-model-family-expands/))",
      "creator_organization": "Google",
      "access": "limited",
      "todo": false,
      "release_date": "2025-07-22"
    },
    {
      "name": "google/gemini-2.5-flash-thinking-disabled",
      "display_name": "Gemini 2.5 Flash (thinking disabled)",
      "short_display_name": "Gemini 2.5 Flash (thinking disabled)",
      "description": "Gemini 2.5 Flash with thinking disabled ([documentation](https://ai.google.dev/gemini-api/docs/models/gemini))",
      "creator_organization": "Google",
      "access": "limited",
      "todo": false,
      "release_date": "2025-06-17"
    },
    {
      "name": "google/gemma-4-31b-it-thinking-disabled",
      "display_name": "Gemma 4 31B Instruct",
      "short_display_name": "Gemma 4 31B Instruct",
      "description": "Gemma is a family of open models built by Google DeepMind. Gemma 4 models are multimodal, handling text and image input (with audio supported on small models) and generating text output. This model was run with thinking disabled. ([blog post](https://huggingface.co/google/gemma-4-31B-it))",
      "creator_organization": "Google",
      "access": "open",
      "todo": false,
      "release_date": "2026-04-02",
      "num_parameters": 33000000000
    },
    {
      "name": "meta/llama-3.3-70b-instruct-turbo",
      "display_name": "Llama 3.3 Instruct Turbo (70B)",
      "short_display_name": "Llama 3.3 Instruct Turbo (70B)",
      "description": "Llama 3.3 (70B) is part of the Llama 3 family of dense Transformer models that that natively support multilinguality, coding, reasoning, and tool usage. ([paper](https://ai.meta.com/research/publications/the-llama-3-herd-of-models/), [documentation](https://www.llama.com/docs/model-cards-and-prompt-formats/llama3_3/), [model card](https://github.com/meta-llama/llama-models/blob/main/models/llama3_3/MODEL_CARD.md)) Turbo is Together's implementation, providing a near negligible difference in quality from the reference implementation with faster performance and lower cost, currently using FP8 quantization. ([blog](https://www.together.ai/blog/llama-31-quality))",
      "creator_organization": "Meta",
      "access": "open",
      "todo": false,
      "release_date": "2024-12-06",
      "num_parameters": 70000000000
    },
    {
      "name": "meta/llama-4-scout-17b-16e-instruct",
      "display_name": "Llama 4 Scout (17Bx16E) Instruct",
      "short_display_name": "Llama 4 Scout (17Bx16E) Instruct",
      "description": "Llama 4 Scout (17Bx16E) Instruct is part of the Llama 4 collection of models are natively multimodal AI models that enable text and multimodal experiences using a mixture-of-experts architecture. ([blog](https://ai.meta.com/blog/llama-4-multimodal-intelligence/))",
      "creator_organization": "Meta",
      "access": "open",
      "todo": false,
      "release_date": "2025-04-05",
      "num_parameters": 109000000000
    },
    {
      "name": "meta/llama-4-maverick-17b-128e-instruct-fp8",
      "display_name": "Llama 4 Maverick (17Bx128E) Instruct FP8",
      "short_display_name": "Llama 4 Maverick (17Bx128E) Instruct FP8",
      "description": "Llama 4 Maverick (17Bx128E) Instruct FP8 is part of the Llama 4 collection of models are natively multimodal AI models that enable text and multimodal experiences using a mixture-of-experts architecture. ([blog](https://ai.meta.com/blog/llama-4-multimodal-intelligence/))",
      "creator_organization": "Meta",
      "access": "open",
      "todo": false,
      "release_date": "2025-04-05",
      "num_parameters": 402000000000
    },
    {
      "name": "mistralai/mistral-large-2411",
      "display_name": "Mistral Large (2411)",
      "short_display_name": "Mistral Large (2411)",
      "description": "Mistral Large (2411) is a 123B parameter model that has a 128k context window. ([blog](https://mistral.ai/news/pixtral-large/))",
      "creator_organization": "Mistral AI",
      "access": "open",
      "todo": false,
      "release_date": "2024-11-18",
      "num_parameters": 123000000000
    },
    {
      "name": "mistralai/mistral-large-2512",
      "display_name": "Mistral Large 3 (2512)",
      "short_display_name": "Mistral Large 3 (2512)",
      "description": "Mistral Large 3 is a open-weight, general-purpose multimodal model with a granular Mixture-of-Experts architecture with 41B active parameters and 675B total parameters. ([doc](https://docs.mistral.ai/models/mistral-large-3-25-12), [blog](https://mistral.ai/news/mistral-3))",
      "creator_organization": "Mistral AI",
      "access": "open",
      "todo": false,
      "release_date": "2025-12-02",
      "num_parameters": 675000000000
    },
    {
      "name": "openai/gpt-4.1-2025-04-14",
      "display_name": "GPT-4.1 (2025-04-14)",
      "short_display_name": "GPT-4.1 (2025-04-14)",
      "description": "GPT-4.1 (2025-04-14) is a multimdodal model in the GPT-4.1 family, which outperforms the GPT-4o family, with major gains in coding and instruction following. They also have larger context windows of 1 million tokens and are able to better use that context with improved long-context comprehension. ([blog](https://openai.com/index/gpt-4-1/))",
      "creator_organization": "OpenAI",
      "access": "limited",
      "todo": false,
      "release_date": "2025-04-14"
    },
    {
      "name": "openai/gpt-4.1-mini-2025-04-14",
      "display_name": "GPT-4.1 mini (2025-04-14)",
      "short_display_name": "GPT-4.1 mini (2025-04-14)",
      "description": "GPT-4.1 mini (2025-04-14) is a multimdodal model in the GPT-4.1 family, which outperforms the GPT-4o family, with major gains in coding and instruction following. They also have larger context windows of 1 million tokens and are able to better use that context with improved long-context comprehension. ([blog](https://openai.com/index/gpt-4-1/))",
      "creator_organization": "OpenAI",
      "access": "limited",
      "todo": false,
      "release_date": "2025-04-14"
    },
    {
      "name": "openai/gpt-4.1-nano-2025-04-14",
      "display_name": "GPT-4.1 nano (2025-04-14)",
      "short_display_name": "GPT-4.1 nano (2025-04-14)",
      "description": "GPT-4.1 nano (2025-04-14) is a multimdodal model in the GPT-4.1 family, which outperforms the GPT-4o family, with major gains in coding and instruction following. They also have larger context windows of 1 million tokens and are able to better use that context with improved long-context comprehension. ([blog](https://openai.com/index/gpt-4-1/))",
      "creator_organization": "OpenAI",
      "access": "limited",
      "todo": false,
      "release_date": "2025-04-14"
    },
    {
      "name": "openai/gpt-5.4-2026-03-05",
      "display_name": "GPT-5.4 (2026-03-05)",
      "short_display_name": "GPT-5.4 (2026-03-05)",
      "description": "GPT-5.4 (2026-03-05) is a model in the GPT-5 model family that incorporates recent advances in reasoning, coding, and agentic workflows. ([blog](https://openai.com/index/introducing-gpt-5-4/))",
      "creator_organization": "OpenAI",
      "access": "limited",
      "todo": false,
      "release_date": "2026-03-05"
    },
    {
      "name": "openai/gpt-5.4-mini-2026-03-17",
      "display_name": "GPT-5.4 mini (2026-03-17)",
      "short_display_name": "GPT-5.4 mini (2026-03-17)",
      "description": "GPT-5.4 mini (2026-03-17) is a more efficient model designed for high-volume workloads. ([blog](https://openai.com/index/introducing-gpt-5-4-mini-and-nano/))",
      "creator_organization": "OpenAI",
      "access": "limited",
      "todo": false,
      "release_date": "2026-03-17"
    },
    {
      "name": "openai/gpt-5.4-nano-2026-03-17",
      "display_name": "GPT-5.4 nano (2026-03-17)",
      "short_display_name": "GPT-5.4 nano (2026-03-17)",
      "description": "GPT-5.4 nano (2026-03-17) is a more efficient model designed for high-volume workloads. ([blog](https://openai.com/index/introducing-gpt-5-4-mini-and-nano/))",
      "creator_organization": "OpenAI",
      "access": "limited",
      "todo": false,
      "release_date": "2026-03-17"
    },
    {
      "name": "openai/gpt-5.1-2025-11-13",
      "display_name": "GPT-5.1 (2025-11-13)",
      "short_display_name": "GPT-5.1 (2025-11-13)",
      "description": "GPT-5.1 (2025-11-13) is a model in the GPT-5 model family, and has similar training for code generation, bug fixing, refactoring, instruction following, long context and tool calling. ([blog](https://openai.com/index/gpt-5-1-for-developers/))",
      "creator_organization": "OpenAI",
      "access": "limited",
      "todo": false,
      "release_date": "2025-11-13"
    },
    {
      "name": "qwen/qwen2.5-7b-instruct-turbo",
      "display_name": "Qwen2.5 Instruct Turbo (7B)",
      "short_display_name": "Qwen2.5 Instruct Turbo (7B)",
      "description": "Qwen2.5 Instruct Turbo (7B) was trained on 18 trillion tokens and supports 29 languages, and shows improvements over Qwen2 in knowledge, coding, mathematics, instruction following, generating long texts, and processing structure data. ([blog](https://qwenlm.github.io/blog/qwen2.5/)) Turbo is Together's cost-efficient implementation, providing fast FP8 performance while maintaining quality, closely matching FP16 reference models. ([blog](https://www.together.ai/blog/together-inference-engine-2))",
      "creator_organization": "Qwen",
      "access": "open",
      "todo": false,
      "release_date": "2024-09-19"
    },
    {
      "name": "qwen/qwen2.5-72b-instruct-turbo",
      "display_name": "Qwen2.5 Instruct Turbo (72B)",
      "short_display_name": "Qwen2.5 Instruct Turbo (72B)",
      "description": "Qwen2.5 Instruct Turbo (72B) was trained on 18 trillion tokens and supports 29 languages, and shows improvements over Qwen2 in knowledge, coding, mathematics, instruction following, generating long texts, and processing structure data. ([blog](https://qwenlm.github.io/blog/qwen2.5/)) Turbo is Together's cost-efficient implementation, providing fast FP8 performance while maintaining quality, closely matching FP16 reference models. ([blog](https://www.together.ai/blog/together-inference-engine-2))",
      "creator_organization": "Qwen",
      "access": "open",
      "todo": false,
      "release_date": "2024-09-19"
    },
    {
      "name": "qwen/qwen3-next-80b-a3b-instruct",
      "display_name": "Qwen3-Next 80B A3B Instruct",
      "short_display_name": "Qwen3-Next 80B A3B Instruct",
      "description": "Qwen3-Next is a new model architecture for improving training and inference efficiency under long-context and large-parameter settings. Compared to the MoE structure of Qwen3, Qwen3-Next introduces a hybrid attention mechanism, a highly sparse Mixture-of-Experts (MoE) structure, training-stability-friendly optimizations, and a multi-token prediction mechanism for faster inference. ([blog](https://qwen.ai/blog?id=4074cca80393150c248e508aa62983f9cb7d27cd&from=research.latest-advancements-list))",
      "creator_organization": "Qwen",
      "access": "open",
      "todo": false,
      "release_date": "2025-09-10",
      "num_parameters": 81000000000
    },
    {
      "name": "qwen/qwen3-235b-a22b-instruct-2507-fp8",
      "display_name": "Qwen3 235B A22B Instruct 2507 FP8",
      "short_display_name": "Qwen3 235B A22B Instruct 2507 FP8",
      "description": "Qwen3 235B A22B Instruct 2507 FP8 is an updated version of the non-thinking mode of Qwen3 235B A22B FP8.",
      "creator_organization": "Qwen",
      "access": "open",
      "todo": false,
      "release_date": "2025-07-21",
      "num_parameters": 235000000000
    },
    {
      "name": "qwen/qwen3.5-397b-a17b-thinking-disabled",
      "display_name": "Qwen3.5 397B A17B",
      "short_display_name": "Qwen3.5 397B A17B",
      "description": "Qwen3.5 397B A17B uses a hybrid architecture that fuses linear attention (via Gated Delta Networks) with a sparse mixture-of-experts. This model was run with thinking disabled. ([blog](https://qwen.ai/blog?id=qwen3.5))",
      "creator_organization": "Qwen",
      "access": "open",
      "todo": false,
      "release_date": "2026-02-15",
      "num_parameters": 403000000000
    },
    {
      "name": "qwen/qwen3.5-9b-thinking-disabled",
      "display_name": "Qwen3.5 9B",
      "short_display_name": "Qwen3.5 9B",
      "description": "Qwen3.5 9B A17B uses a hybrid architecture that fuses linear attention (via Gated Delta Networks) with a sparse mixture-of-experts. This model was run with thinking disabled. ([blog](https://qwen.ai/blog?id=qwen3.5))",
      "creator_organization": "Qwen",
      "access": "open",
      "todo": false,
      "release_date": "2026-03-02",
      "num_parameters": 10000000000
    },
    {
      "name": "tiiuae/falcon3-7b-instruct",
      "display_name": "Falcon3-7B-Instruct",
      "short_display_name": "Falcon3-7B-Instruct",
      "description": "Falcon3-7B-Instruct is an open-weights foundation model that supports 4 languages (English, French, Spanish, Portuguese) that was trained on 14T tokens. ([blog](https://falconllm.tii.ae/falcon3/index.html))",
      "creator_organization": "TII UAE",
      "access": "open",
      "todo": false,
      "release_date": "2024-12-17",
      "num_parameters": 7460000000
    },
    {
      "name": "tiiuae/falcon3-10b-instruct",
      "display_name": "Falcon3-10B-Instruct",
      "short_display_name": "Falcon3-10B-Instruct",
      "description": "Falcon3-10B-Instruct is an open-weights foundation model that supports 4 languages (English, French, Spanish, Portuguese) that was trained on 14T tokens. ([blog](https://falconllm.tii.ae/falcon3/index.html))",
      "creator_organization": "TII UAE",
      "access": "open",
      "todo": false,
      "release_date": "2024-12-17",
      "num_parameters": 10300000000
    },
    {
      "name": "freedomintelligence/acegpt-v2-8b-chat",
      "display_name": "AceGPT-v2-8B-Chat",
      "short_display_name": "AceGPT-v2-8B-Chat",
      "description": "AceGPT is a fully fine-tuned generative text model collection, particularly focused on the Arabic language domain. AceGPT-v2-8B-Chat is based on Meta-Llama-3-8B. ([paper](https://arxiv.org/abs/2412.03253))",
      "creator_organization": "FreedomAI",
      "access": "open",
      "todo": false,
      "release_date": "2024-10-20",
      "num_parameters": 8030000000
    },
    {
      "name": "freedomintelligence/acegpt-v2-32b-chat",
      "display_name": "AceGPT-v2-32B-Chat",
      "short_display_name": "AceGPT-v2-32B-Chat",
      "description": "AceGPT is a fully fine-tuned generative text model collection, particularly focused on the Arabic language domain. AceGPT-v2-32B-Chat is based on Qwen1.5-32B. ([paper](https://arxiv.org/abs/2412.03253))",
      "creator_organization": "FreedomAI",
      "access": "open",
      "todo": false,
      "release_date": "2024-10-20",
      "num_parameters": 32500000000
    },
    {
      "name": "freedomintelligence/acegpt-v2-70b-chat",
      "display_name": "AceGPT-v2-70B-Chat",
      "short_display_name": "AceGPT-v2-70B-Chat",
      "description": "AceGPT is a fully fine-tuned generative text model collection, particularly focused on the Arabic language domain. AceGPT-v2-70B-Chat is based on Meta-Llama-3-70B. ([paper](https://arxiv.org/abs/2412.03253))",
      "creator_organization": "FreedomAI",
      "access": "open",
      "todo": false,
      "release_date": "2024-10-20",
      "num_parameters": 70600000000
    },
    {
      "name": "allam-ai/allam-7b-instruct-preview",
      "display_name": "ALLaM-7B-Instruct-preview",
      "short_display_name": "ALLaM-7B-Instruct-preview",
      "description": "ALLaM-7B-Instruct-preview is a model designed to advance Arabic language technology, which used a recipe of training on 4T English tokens followed by training on 1.2T mixed Arabic/English tokens. ([paper](https://arxiv.org/abs/2407.15390v1))",
      "creator_organization": "NCAI & SDAIA",
      "access": "open",
      "todo": false,
      "release_date": "2024-07-22",
      "num_parameters": 7000000000
    },
    {
      "name": "silma-ai/silma-9b-instruct-v1.0",
      "display_name": "SILMA 9B",
      "short_display_name": "SILMA 9B",
      "description": "SILMA 9B is a compact Arabic language model based on Google Gemma. ([model card](https://huggingface.co/silma-ai/SILMA-9B-Instruct-v1.0))",
      "creator_organization": "SILMA AI",
      "access": "open",
      "todo": false,
      "release_date": "2024-08-17",
      "num_parameters": 9240000000
    },
    {
      "name": "inceptionai/jais-family-30b-16k-chat",
      "display_name": "Jais-family-30b-16k-chat",
      "short_display_name": "Jais-family-30b-16k-chat",
      "description": "The Jais family of models is a series of bilingual English-Arabic large language models (LLMs) that are trained from scratch and optimized to excel in Arabic while having strong English capabilities. ([website](https://inceptionai.ai/jaisfamily/index.html), [blog](https://mbzuai.ac.ae/news/meet-jais-the-worlds-most-advanced-arabic-large-language-model-open-sourced-by-g42s-inception/))",
      "creator_organization": "Inception",
      "access": "open",
      "todo": false,
      "release_date": "2023-08-30",
      "num_parameters": 30800000000
    },
    {
      "name": "inceptionai/jais-adapted-7b-chat",
      "display_name": "Jais-adapted-7b-chat",
      "short_display_name": "Jais-adapted-7b-chat",
      "description": "The Jais adapted models are bilingual English-Arabic large language models (LLMs) that are trained adaptively from Llama-2 and optimized to excel in Arabic while having strong English capabilities. ([website](https://inceptionai.ai/jaisfamily/index.html), [blog](https://mbzuai.ac.ae/news/meet-jais-the-worlds-most-advanced-arabic-large-language-model-open-sourced-by-g42s-inception/))",
      "creator_organization": "Inception",
      "access": "open",
      "todo": false,
      "release_date": "2023-08-30",
      "num_parameters": 7000000000
    },
    {
      "name": "inceptionai/jais-adapted-13b-chat",
      "display_name": "Jais-adapted-13b-chat",
      "short_display_name": "Jais-adapted-13b-chat",
      "description": "The Jais adapted models are bilingual English-Arabic large language models (LLMs) that are trained adaptively from Llama-2 and optimized to excel in Arabic while having strong English capabilities. ([website](https://inceptionai.ai/jaisfamily/index.html), [blog](https://mbzuai.ac.ae/news/meet-jais-the-worlds-most-advanced-arabic-large-language-model-open-sourced-by-g42s-inception/))",
      "creator_organization": "Inception",
      "access": "open",
      "todo": false,
      "release_date": "2023-08-30",
      "num_parameters": 13300000000
    },
    {
      "name": "inceptionai/jais-adapted-70b-chat",
      "display_name": "Jais-adapted-70b-chat",
      "short_display_name": "Jais-adapted-70b-chat",
      "description": "The Jais adapted models are bilingual English-Arabic large language models (LLMs) that are trained adaptively from Llama-2 and optimized to excel in Arabic while having strong English capabilities. ([website](https://inceptionai.ai/jaisfamily/index.html), [blog](https://mbzuai.ac.ae/news/meet-jais-the-worlds-most-advanced-arabic-large-language-model-open-sourced-by-g42s-inception/))",
      "creator_organization": "Inception",
      "access": "open",
      "todo": false,
      "release_date": "2023-08-30",
      "num_parameters": 69500000000
    },
    {
      "name": "arabicai/llm-x-2026-06-07",
      "display_name": "Arabic.AI LLM-X (2026-06-07)",
      "short_display_name": "Arabic.AI LLM-X (2026-06-07)",
      "description": "Arabic.AI LLM-X (2026-06-07) is a flagship model, designed with Arabic at its core and optimized to deliver enterprise-grade performance, strong agentic capabilities, and scalability for complex workloads.",
      "creator_organization": "Arabic.AI",
      "access": "limited",
      "todo": false
    },
    {
      "name": "local/incept-arabic-1",
      "display_name": "incept-arabic-1",
      "short_display_name": "incept-arabic-1",
      "description": "incept-arabic-1 is a proprietary large language model developed by Incept Labs, a Titan Holdings portfolio company. The model is post-trained on a curated corpus of Modern Standard Arabic and regional dialects to deliver state-of-the-art performance on Arabic comprehension, reasoning and generation.",
      "creator_organization": "Incept Labs",
      "access": "closed",
      "todo": false
    }
  ]
}