[ { "title": "Instruction Following", "header": [ { "value": "Model/adapter", "markdown": false, "metadata": {} }, { "value": "Mean win rate", "description": "How many models this model outperform on average (over columns).", "markdown": false, "lower_is_better": false, "metadata": {} }, { "value": "Anthropic RLHF dataset - Helpfulness", "description": "The dialogue datasets released by Anthropic to facilitate research in model helpfulness and harmlessness ([Bai et al., 2022](https://arxiv.org/pdf/2204.05862.pdf); [Ganguli et al., 2022](https://arxiv.org/pdf/2209.07858.pdf)). We only use the first utterance of each dialogue.\n\nHelpfulness: Whether the model appears to do what it is instructed to.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Helpfulness", "run_group": "Anthropic RLHF dataset" } }, { "value": "Anthropic RLHF dataset - Understandability", "description": "The dialogue datasets released by Anthropic to facilitate research in model helpfulness and harmlessness ([Bai et al., 2022](https://arxiv.org/pdf/2204.05862.pdf); [Ganguli et al., 2022](https://arxiv.org/pdf/2209.07858.pdf)). We only use the first utterance of each dialogue.\n\nUnderstandability: Whether the answer is easy to comprehend.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Understandability", "run_group": "Anthropic RLHF dataset" } }, { "value": "Anthropic RLHF dataset - Completeness", "description": "The dialogue datasets released by Anthropic to facilitate research in model helpfulness and harmlessness ([Bai et al., 2022](https://arxiv.org/pdf/2204.05862.pdf); [Ganguli et al., 2022](https://arxiv.org/pdf/2209.07858.pdf)). We only use the first utterance of each dialogue.\n\nCompleteness: Whether the response provides a comprehensive answer to the question or request.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Completeness", "run_group": "Anthropic RLHF dataset" } }, { "value": "Anthropic RLHF dataset - Conciseness", "description": "The dialogue datasets released by Anthropic to facilitate research in model helpfulness and harmlessness ([Bai et al., 2022](https://arxiv.org/pdf/2204.05862.pdf); [Ganguli et al., 2022](https://arxiv.org/pdf/2209.07858.pdf)). We only use the first utterance of each dialogue.\n\nConciseness: Whether the response is concise and to the point, without unnecessary repetition or elaboration.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Conciseness", "run_group": "Anthropic RLHF dataset" } }, { "value": "Anthropic RLHF dataset - Harmlessness", "description": "The dialogue datasets released by Anthropic to facilitate research in model helpfulness and harmlessness ([Bai et al., 2022](https://arxiv.org/pdf/2204.05862.pdf); [Ganguli et al., 2022](https://arxiv.org/pdf/2209.07858.pdf)). We only use the first utterance of each dialogue.\n\nHarmlessness: Whether the response avoids using toxic language or helping the user with harmful goals.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Harmlessness", "run_group": "Anthropic RLHF dataset" } }, { "value": "Best ChatGPT Prompts - Helpfulness", "description": "A list of \u201cbest ChatGPT prompts to power your workflow\u201d summarized by [GRIDFITI](https://gridfiti.com/best-chatgpt-prompts/).\n\nHelpfulness: Whether the model appears to do what it is instructed to.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Helpfulness", "run_group": "Best ChatGPT Prompts" } }, { "value": "Best ChatGPT Prompts - Understandability", "description": "A list of \u201cbest ChatGPT prompts to power your workflow\u201d summarized by [GRIDFITI](https://gridfiti.com/best-chatgpt-prompts/).\n\nUnderstandability: Whether the answer is easy to comprehend.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Understandability", "run_group": "Best ChatGPT Prompts" } }, { "value": "Best ChatGPT Prompts - Completeness", "description": "A list of \u201cbest ChatGPT prompts to power your workflow\u201d summarized by [GRIDFITI](https://gridfiti.com/best-chatgpt-prompts/).\n\nCompleteness: Whether the response provides a comprehensive answer to the question or request.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Completeness", "run_group": "Best ChatGPT Prompts" } }, { "value": "Best ChatGPT Prompts - Conciseness", "description": "A list of \u201cbest ChatGPT prompts to power your workflow\u201d summarized by [GRIDFITI](https://gridfiti.com/best-chatgpt-prompts/).\n\nConciseness: Whether the response is concise and to the point, without unnecessary repetition or elaboration.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Conciseness", "run_group": "Best ChatGPT Prompts" } }, { "value": "Best ChatGPT Prompts - Harmlessness", "description": "A list of \u201cbest ChatGPT prompts to power your workflow\u201d summarized by [GRIDFITI](https://gridfiti.com/best-chatgpt-prompts/).\n\nHarmlessness: Whether the response avoids using toxic language or helping the user with harmful goals.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Harmlessness", "run_group": "Best ChatGPT Prompts" } }, { "value": "Koala test dataset - Helpfulness", "description": "The test dataset from the [Koala paper](https://bair.berkeley.edu/blog/2023/04/03/koala/) for evaluating instruction-following models.\n\nHelpfulness: Whether the model appears to do what it is instructed to.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Helpfulness", "run_group": "Koala test dataset" } }, { "value": "Koala test dataset - Understandability", "description": "The test dataset from the [Koala paper](https://bair.berkeley.edu/blog/2023/04/03/koala/) for evaluating instruction-following models.\n\nUnderstandability: Whether the answer is easy to comprehend.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Understandability", "run_group": "Koala test dataset" } }, { "value": "Koala test dataset - Completeness", "description": "The test dataset from the [Koala paper](https://bair.berkeley.edu/blog/2023/04/03/koala/) for evaluating instruction-following models.\n\nCompleteness: Whether the response provides a comprehensive answer to the question or request.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Completeness", "run_group": "Koala test dataset" } }, { "value": "Koala test dataset - Conciseness", "description": "The test dataset from the [Koala paper](https://bair.berkeley.edu/blog/2023/04/03/koala/) for evaluating instruction-following models.\n\nConciseness: Whether the response is concise and to the point, without unnecessary repetition or elaboration.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Conciseness", "run_group": "Koala test dataset" } }, { "value": "Koala test dataset - Harmlessness", "description": "The test dataset from the [Koala paper](https://bair.berkeley.edu/blog/2023/04/03/koala/) for evaluating instruction-following models.\n\nHarmlessness: Whether the response avoids using toxic language or helping the user with harmful goals.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Harmlessness", "run_group": "Koala test dataset" } }, { "value": "Open Assistant - Helpfulness", "description": "LAION\u2019s OpenAssistant Conversations Dataset (OASST1) that consists of 66,497 conversation trees ([K\u00f6pf et al., 2023](https://openreview.net/forum?id=VSJotgbPHF)). We only use the initial prompt in each conversation.\n\nHelpfulness: Whether the model appears to do what it is instructed to.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Helpfulness", "run_group": "Open Assistant" } }, { "value": "Open Assistant - Understandability", "description": "LAION\u2019s OpenAssistant Conversations Dataset (OASST1) that consists of 66,497 conversation trees ([K\u00f6pf et al., 2023](https://openreview.net/forum?id=VSJotgbPHF)). We only use the initial prompt in each conversation.\n\nUnderstandability: Whether the answer is easy to comprehend.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Understandability", "run_group": "Open Assistant" } }, { "value": "Open Assistant - Completeness", "description": "LAION\u2019s OpenAssistant Conversations Dataset (OASST1) that consists of 66,497 conversation trees ([K\u00f6pf et al., 2023](https://openreview.net/forum?id=VSJotgbPHF)). We only use the initial prompt in each conversation.\n\nCompleteness: Whether the response provides a comprehensive answer to the question or request.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Completeness", "run_group": "Open Assistant" } }, { "value": "Open Assistant - Conciseness", "description": "LAION\u2019s OpenAssistant Conversations Dataset (OASST1) that consists of 66,497 conversation trees ([K\u00f6pf et al., 2023](https://openreview.net/forum?id=VSJotgbPHF)). We only use the initial prompt in each conversation.\n\nConciseness: Whether the response is concise and to the point, without unnecessary repetition or elaboration.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Conciseness", "run_group": "Open Assistant" } }, { "value": "Open Assistant - Harmlessness", "description": "LAION\u2019s OpenAssistant Conversations Dataset (OASST1) that consists of 66,497 conversation trees ([K\u00f6pf et al., 2023](https://openreview.net/forum?id=VSJotgbPHF)). We only use the initial prompt in each conversation.\n\nHarmlessness: Whether the response avoids using toxic language or helping the user with harmful goals.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Harmlessness", "run_group": "Open Assistant" } }, { "value": "Self Instruct - Helpfulness", "description": "The manually-curated instructions from the Self-Instruct paper ([Wang et al., 2023](https://aclanthology.org/2023.acl-long.754.pdf)).\n\nHelpfulness: Whether the model appears to do what it is instructed to.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Helpfulness", "run_group": "Self Instruct" } }, { "value": "Self Instruct - Understandability", "description": "The manually-curated instructions from the Self-Instruct paper ([Wang et al., 2023](https://aclanthology.org/2023.acl-long.754.pdf)).\n\nUnderstandability: Whether the answer is easy to comprehend.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Understandability", "run_group": "Self Instruct" } }, { "value": "Self Instruct - Completeness", "description": "The manually-curated instructions from the Self-Instruct paper ([Wang et al., 2023](https://aclanthology.org/2023.acl-long.754.pdf)).\n\nCompleteness: Whether the response provides a comprehensive answer to the question or request.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Completeness", "run_group": "Self Instruct" } }, { "value": "Self Instruct - Conciseness", "description": "The manually-curated instructions from the Self-Instruct paper ([Wang et al., 2023](https://aclanthology.org/2023.acl-long.754.pdf)).\n\nConciseness: Whether the response is concise and to the point, without unnecessary repetition or elaboration.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Conciseness", "run_group": "Self Instruct" } }, { "value": "Self Instruct - Harmlessness", "description": "The manually-curated instructions from the Self-Instruct paper ([Wang et al., 2023](https://aclanthology.org/2023.acl-long.754.pdf)).\n\nHarmlessness: Whether the response avoids using toxic language or helping the user with harmful goals.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Harmlessness", "run_group": "Self Instruct" } }, { "value": "Vicuna - Helpfulness", "description": "The set of prompts used by the [Vicuna](https://lmsys.org/blog/2023-03-30-vicuna/) team to evaluate instruction-following models.\n\nHelpfulness: Whether the model appears to do what it is instructed to.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Helpfulness", "run_group": "Vicuna" } }, { "value": "Vicuna - Understandability", "description": "The set of prompts used by the [Vicuna](https://lmsys.org/blog/2023-03-30-vicuna/) team to evaluate instruction-following models.\n\nUnderstandability: Whether the answer is easy to comprehend.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Understandability", "run_group": "Vicuna" } }, { "value": "Vicuna - Completeness", "description": "The set of prompts used by the [Vicuna](https://lmsys.org/blog/2023-03-30-vicuna/) team to evaluate instruction-following models.\n\nCompleteness: Whether the response provides a comprehensive answer to the question or request.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Completeness", "run_group": "Vicuna" } }, { "value": "Vicuna - Conciseness", "description": "The set of prompts used by the [Vicuna](https://lmsys.org/blog/2023-03-30-vicuna/) team to evaluate instruction-following models.\n\nConciseness: Whether the response is concise and to the point, without unnecessary repetition or elaboration.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Conciseness", "run_group": "Vicuna" } }, { "value": "Vicuna - Harmlessness", "description": "The set of prompts used by the [Vicuna](https://lmsys.org/blog/2023-03-30-vicuna/) team to evaluate instruction-following models.\n\nHarmlessness: Whether the response avoids using toxic language or helping the user with harmful goals.", "markdown": false, "lower_is_better": false, "metadata": { "metric": "Harmlessness", "run_group": "Vicuna" } } ], "rows": [ [ { "value": "Anthropic Claude v1.3", "description": "", "markdown": false }, { "value": 0.611111111111111, "markdown": false }, { "value": 3.6418749999999998, "description": "min=2.81, mean=3.642, max=4.375, sum=29.135 (8)", "style": {}, "markdown": false, "run_spec_names": [ "anthropic_hh_rlhf:subset=hh,model=anthropic_claude-v1.3,evaluator=claude", "anthropic_hh_rlhf:subset=hh,model=anthropic_claude-v1.3,evaluator=gpt4", "anthropic_hh_rlhf:subset=hh,model=anthropic_claude-v1.3,evaluator=mturk", "anthropic_hh_rlhf:subset=hh,model=anthropic_claude-v1.3,evaluator=scale", "anthropic_hh_rlhf:subset=red_team,model=anthropic_claude-v1.3,evaluator=claude", "anthropic_hh_rlhf:subset=red_team,model=anthropic_claude-v1.3,evaluator=gpt4", "anthropic_hh_rlhf:subset=red_team,model=anthropic_claude-v1.3,evaluator=mturk", "anthropic_hh_rlhf:subset=red_team,model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.97625, "description": "min=4.915, mean=4.976, max=5, sum=39.81 (8)", "style": {}, "markdown": false, "run_spec_names": [ "anthropic_hh_rlhf:subset=hh,model=anthropic_claude-v1.3,evaluator=claude", "anthropic_hh_rlhf:subset=hh,model=anthropic_claude-v1.3,evaluator=gpt4", "anthropic_hh_rlhf:subset=hh,model=anthropic_claude-v1.3,evaluator=mturk", "anthropic_hh_rlhf:subset=hh,model=anthropic_claude-v1.3,evaluator=scale", "anthropic_hh_rlhf:subset=red_team,model=anthropic_claude-v1.3,evaluator=claude", "anthropic_hh_rlhf:subset=red_team,model=anthropic_claude-v1.3,evaluator=gpt4", "anthropic_hh_rlhf:subset=red_team,model=anthropic_claude-v1.3,evaluator=mturk", "anthropic_hh_rlhf:subset=red_team,model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 3.8825, "description": "min=2.52, mean=3.882, max=4.87, sum=31.06 (8)", "style": {}, "markdown": false, "run_spec_names": [ "anthropic_hh_rlhf:subset=hh,model=anthropic_claude-v1.3,evaluator=claude", "anthropic_hh_rlhf:subset=hh,model=anthropic_claude-v1.3,evaluator=gpt4", "anthropic_hh_rlhf:subset=hh,model=anthropic_claude-v1.3,evaluator=mturk", "anthropic_hh_rlhf:subset=hh,model=anthropic_claude-v1.3,evaluator=scale", "anthropic_hh_rlhf:subset=red_team,model=anthropic_claude-v1.3,evaluator=claude", "anthropic_hh_rlhf:subset=red_team,model=anthropic_claude-v1.3,evaluator=gpt4", "anthropic_hh_rlhf:subset=red_team,model=anthropic_claude-v1.3,evaluator=mturk", "anthropic_hh_rlhf:subset=red_team,model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.141249999999999, "description": "min=3.7, mean=4.141, max=4.805, sum=33.13 (8)", "style": {}, "markdown": false, "run_spec_names": [ "anthropic_hh_rlhf:subset=hh,model=anthropic_claude-v1.3,evaluator=claude", "anthropic_hh_rlhf:subset=hh,model=anthropic_claude-v1.3,evaluator=gpt4", "anthropic_hh_rlhf:subset=hh,model=anthropic_claude-v1.3,evaluator=mturk", "anthropic_hh_rlhf:subset=hh,model=anthropic_claude-v1.3,evaluator=scale", "anthropic_hh_rlhf:subset=red_team,model=anthropic_claude-v1.3,evaluator=claude", "anthropic_hh_rlhf:subset=red_team,model=anthropic_claude-v1.3,evaluator=gpt4", "anthropic_hh_rlhf:subset=red_team,model=anthropic_claude-v1.3,evaluator=mturk", "anthropic_hh_rlhf:subset=red_team,model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.965, "description": "min=4.925, mean=4.965, max=5, sum=39.72 (8)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "anthropic_hh_rlhf:subset=hh,model=anthropic_claude-v1.3,evaluator=claude", "anthropic_hh_rlhf:subset=hh,model=anthropic_claude-v1.3,evaluator=gpt4", "anthropic_hh_rlhf:subset=hh,model=anthropic_claude-v1.3,evaluator=mturk", "anthropic_hh_rlhf:subset=hh,model=anthropic_claude-v1.3,evaluator=scale", "anthropic_hh_rlhf:subset=red_team,model=anthropic_claude-v1.3,evaluator=claude", "anthropic_hh_rlhf:subset=red_team,model=anthropic_claude-v1.3,evaluator=gpt4", "anthropic_hh_rlhf:subset=red_team,model=anthropic_claude-v1.3,evaluator=mturk", "anthropic_hh_rlhf:subset=red_team,model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.2250000000000005, "description": "min=3.93, mean=4.225, max=4.595, sum=16.9 (4)", "style": {}, "markdown": false, "run_spec_names": [ "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=anthropic_claude-v1.3,evaluator=claude", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=anthropic_claude-v1.3,evaluator=gpt4", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=anthropic_claude-v1.3,evaluator=mturk", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.93625, "description": "min=4.87, mean=4.936, max=4.98, sum=19.745 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=anthropic_claude-v1.3,evaluator=claude", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=anthropic_claude-v1.3,evaluator=gpt4", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=anthropic_claude-v1.3,evaluator=mturk", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.6775, "description": "min=4.57, mean=4.678, max=4.87, sum=18.71 (4)", "style": {}, "markdown": false, "run_spec_names": [ "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=anthropic_claude-v1.3,evaluator=claude", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=anthropic_claude-v1.3,evaluator=gpt4", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=anthropic_claude-v1.3,evaluator=mturk", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.186249999999999, "description": "min=3.82, mean=4.186, max=4.7, sum=16.745 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=anthropic_claude-v1.3,evaluator=claude", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=anthropic_claude-v1.3,evaluator=gpt4", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=anthropic_claude-v1.3,evaluator=mturk", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.995, "description": "min=4.985, mean=4.995, max=5, sum=19.98 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=anthropic_claude-v1.3,evaluator=claude", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=anthropic_claude-v1.3,evaluator=gpt4", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=anthropic_claude-v1.3,evaluator=mturk", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.178985148514852, "description": "min=3.9, mean=4.179, max=4.48, sum=16.716 (4)", "style": {}, "markdown": false, "run_spec_names": [ "koala:model=anthropic_claude-v1.3,evaluator=claude", "koala:model=anthropic_claude-v1.3,evaluator=gpt4", "koala:model=anthropic_claude-v1.3,evaluator=mturk", "koala:model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.88914603960396, "description": "min=4.735, mean=4.889, max=5, sum=19.557 (4)", "style": {}, "markdown": false, "run_spec_names": [ "koala:model=anthropic_claude-v1.3,evaluator=claude", "koala:model=anthropic_claude-v1.3,evaluator=gpt4", "koala:model=anthropic_claude-v1.3,evaluator=mturk", "koala:model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.6123514851485155, "description": "min=4.52, mean=4.612, max=4.78, sum=18.449 (4)", "style": {}, "markdown": false, "run_spec_names": [ "koala:model=anthropic_claude-v1.3,evaluator=claude", "koala:model=anthropic_claude-v1.3,evaluator=gpt4", "koala:model=anthropic_claude-v1.3,evaluator=mturk", "koala:model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 3.9995792079207924, "description": "min=3.72, mean=4.0, max=4.27, sum=15.998 (4)", "style": {}, "markdown": false, "run_spec_names": [ "koala:model=anthropic_claude-v1.3,evaluator=claude", "koala:model=anthropic_claude-v1.3,evaluator=gpt4", "koala:model=anthropic_claude-v1.3,evaluator=mturk", "koala:model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.981324257425743, "description": "min=4.965, mean=4.981, max=5, sum=19.925 (4)", "style": {}, "markdown": false, "run_spec_names": [ "koala:model=anthropic_claude-v1.3,evaluator=claude", "koala:model=anthropic_claude-v1.3,evaluator=gpt4", "koala:model=anthropic_claude-v1.3,evaluator=mturk", "koala:model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 3.98625, "description": "min=3.7, mean=3.986, max=4.31, sum=15.945 (4)", "style": {}, "markdown": false, "run_spec_names": [ "open_assistant:language=en,model=anthropic_claude-v1.3,evaluator=claude", "open_assistant:language=en,model=anthropic_claude-v1.3,evaluator=gpt4", "open_assistant:language=en,model=anthropic_claude-v1.3,evaluator=mturk", "open_assistant:language=en,model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.91625, "description": "min=4.81, mean=4.916, max=5, sum=19.665 (4)", "style": {}, "markdown": false, "run_spec_names": [ "open_assistant:language=en,model=anthropic_claude-v1.3,evaluator=claude", "open_assistant:language=en,model=anthropic_claude-v1.3,evaluator=gpt4", "open_assistant:language=en,model=anthropic_claude-v1.3,evaluator=mturk", "open_assistant:language=en,model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.4437500000000005, "description": "min=4.335, mean=4.444, max=4.65, sum=17.775 (4)", "style": {}, "markdown": false, "run_spec_names": [ "open_assistant:language=en,model=anthropic_claude-v1.3,evaluator=claude", "open_assistant:language=en,model=anthropic_claude-v1.3,evaluator=gpt4", "open_assistant:language=en,model=anthropic_claude-v1.3,evaluator=mturk", "open_assistant:language=en,model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.0737499999999995, "description": "min=3.66, mean=4.074, max=4.37, sum=16.295 (4)", "style": {}, "markdown": false, "run_spec_names": [ "open_assistant:language=en,model=anthropic_claude-v1.3,evaluator=claude", "open_assistant:language=en,model=anthropic_claude-v1.3,evaluator=gpt4", "open_assistant:language=en,model=anthropic_claude-v1.3,evaluator=mturk", "open_assistant:language=en,model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.975, "description": "min=4.935, mean=4.975, max=5, sum=19.9 (4)", "style": {}, "markdown": false, "run_spec_names": [ "open_assistant:language=en,model=anthropic_claude-v1.3,evaluator=claude", "open_assistant:language=en,model=anthropic_claude-v1.3,evaluator=gpt4", "open_assistant:language=en,model=anthropic_claude-v1.3,evaluator=mturk", "open_assistant:language=en,model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.253836633663367, "description": "min=3.96, mean=4.254, max=4.62, sum=17.015 (4)", "style": {}, "markdown": false, "run_spec_names": [ "self_instruct:model=anthropic_claude-v1.3,evaluator=claude", "self_instruct:model=anthropic_claude-v1.3,evaluator=gpt4", "self_instruct:model=anthropic_claude-v1.3,evaluator=mturk", "self_instruct:model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.935309405940594, "description": "min=4.865, mean=4.935, max=5, sum=19.741 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "self_instruct:model=anthropic_claude-v1.3,evaluator=claude", "self_instruct:model=anthropic_claude-v1.3,evaluator=gpt4", "self_instruct:model=anthropic_claude-v1.3,evaluator=mturk", "self_instruct:model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.717227722772277, "description": "min=4.609, mean=4.717, max=4.91, sum=18.869 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "self_instruct:model=anthropic_claude-v1.3,evaluator=claude", "self_instruct:model=anthropic_claude-v1.3,evaluator=gpt4", "self_instruct:model=anthropic_claude-v1.3,evaluator=mturk", "self_instruct:model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.255940594059406, "description": "min=3.88, mean=4.256, max=4.624, sum=17.024 (4)", "style": {}, "markdown": false, "run_spec_names": [ "self_instruct:model=anthropic_claude-v1.3,evaluator=claude", "self_instruct:model=anthropic_claude-v1.3,evaluator=gpt4", "self_instruct:model=anthropic_claude-v1.3,evaluator=mturk", "self_instruct:model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.9925, "description": "min=4.98, mean=4.992, max=5, sum=19.97 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "self_instruct:model=anthropic_claude-v1.3,evaluator=claude", "self_instruct:model=anthropic_claude-v1.3,evaluator=gpt4", "self_instruct:model=anthropic_claude-v1.3,evaluator=mturk", "self_instruct:model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.18125, "description": "min=3.862, mean=4.181, max=4.556, sum=16.725 (4)", "style": {}, "markdown": false, "run_spec_names": [ "vicuna:category=all,model=anthropic_claude-v1.3,evaluator=claude", "vicuna:category=all,model=anthropic_claude-v1.3,evaluator=gpt4", "vicuna:category=all,model=anthropic_claude-v1.3,evaluator=mturk", "vicuna:category=all,model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.895312499999999, "description": "min=4.694, mean=4.895, max=5, sum=19.581 (4)", "style": {}, "markdown": false, "run_spec_names": [ "vicuna:category=all,model=anthropic_claude-v1.3,evaluator=claude", "vicuna:category=all,model=anthropic_claude-v1.3,evaluator=gpt4", "vicuna:category=all,model=anthropic_claude-v1.3,evaluator=mturk", "vicuna:category=all,model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.6828125, "description": "min=4.569, mean=4.683, max=4.8, sum=18.731 (4)", "style": {}, "markdown": false, "run_spec_names": [ "vicuna:category=all,model=anthropic_claude-v1.3,evaluator=claude", "vicuna:category=all,model=anthropic_claude-v1.3,evaluator=gpt4", "vicuna:category=all,model=anthropic_claude-v1.3,evaluator=mturk", "vicuna:category=all,model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.059375, "description": "min=3.737, mean=4.059, max=4.406, sum=16.238 (4)", "style": {}, "markdown": false, "run_spec_names": [ "vicuna:category=all,model=anthropic_claude-v1.3,evaluator=claude", "vicuna:category=all,model=anthropic_claude-v1.3,evaluator=gpt4", "vicuna:category=all,model=anthropic_claude-v1.3,evaluator=mturk", "vicuna:category=all,model=anthropic_claude-v1.3,evaluator=scale" ] }, { "value": 4.9890625, "description": "min=4.956, mean=4.989, max=5, sum=19.956 (4)", "style": {}, "markdown": false, "run_spec_names": [ "vicuna:category=all,model=anthropic_claude-v1.3,evaluator=claude", "vicuna:category=all,model=anthropic_claude-v1.3,evaluator=gpt4", "vicuna:category=all,model=anthropic_claude-v1.3,evaluator=mturk", "vicuna:category=all,model=anthropic_claude-v1.3,evaluator=scale" ] } ], [ { "value": "Cohere Command beta (52.4B)", "description": "", "markdown": false }, { "value": 0.08888888888888889, "markdown": false }, { "value": 3.523125, "description": "min=2.43, mean=3.523, max=4.21, sum=28.185 (8)", "style": {}, "markdown": false, "run_spec_names": [ "anthropic_hh_rlhf:subset=hh,model=cohere_command-xlarge-beta,evaluator=claude", "anthropic_hh_rlhf:subset=hh,model=cohere_command-xlarge-beta,evaluator=gpt4", "anthropic_hh_rlhf:subset=hh,model=cohere_command-xlarge-beta,evaluator=mturk", "anthropic_hh_rlhf:subset=hh,model=cohere_command-xlarge-beta,evaluator=scale", "anthropic_hh_rlhf:subset=red_team,model=cohere_command-xlarge-beta,evaluator=claude", "anthropic_hh_rlhf:subset=red_team,model=cohere_command-xlarge-beta,evaluator=gpt4", "anthropic_hh_rlhf:subset=red_team,model=cohere_command-xlarge-beta,evaluator=mturk", "anthropic_hh_rlhf:subset=red_team,model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 4.836250000000001, "description": "min=4.68, mean=4.836, max=4.94, sum=38.69 (8)", "style": {}, "markdown": false, "run_spec_names": [ "anthropic_hh_rlhf:subset=hh,model=cohere_command-xlarge-beta,evaluator=claude", "anthropic_hh_rlhf:subset=hh,model=cohere_command-xlarge-beta,evaluator=gpt4", "anthropic_hh_rlhf:subset=hh,model=cohere_command-xlarge-beta,evaluator=mturk", "anthropic_hh_rlhf:subset=hh,model=cohere_command-xlarge-beta,evaluator=scale", "anthropic_hh_rlhf:subset=red_team,model=cohere_command-xlarge-beta,evaluator=claude", "anthropic_hh_rlhf:subset=red_team,model=cohere_command-xlarge-beta,evaluator=gpt4", "anthropic_hh_rlhf:subset=red_team,model=cohere_command-xlarge-beta,evaluator=mturk", "anthropic_hh_rlhf:subset=red_team,model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 3.65625, "description": "min=2.9, mean=3.656, max=4.145, sum=29.25 (8)", "style": {}, "markdown": false, "run_spec_names": [ "anthropic_hh_rlhf:subset=hh,model=cohere_command-xlarge-beta,evaluator=claude", "anthropic_hh_rlhf:subset=hh,model=cohere_command-xlarge-beta,evaluator=gpt4", "anthropic_hh_rlhf:subset=hh,model=cohere_command-xlarge-beta,evaluator=mturk", "anthropic_hh_rlhf:subset=hh,model=cohere_command-xlarge-beta,evaluator=scale", "anthropic_hh_rlhf:subset=red_team,model=cohere_command-xlarge-beta,evaluator=claude", "anthropic_hh_rlhf:subset=red_team,model=cohere_command-xlarge-beta,evaluator=gpt4", "anthropic_hh_rlhf:subset=red_team,model=cohere_command-xlarge-beta,evaluator=mturk", "anthropic_hh_rlhf:subset=red_team,model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 3.9725, "description": "min=3.27, mean=3.973, max=4.545, sum=31.78 (8)", "style": {}, "markdown": false, "run_spec_names": [ "anthropic_hh_rlhf:subset=hh,model=cohere_command-xlarge-beta,evaluator=claude", "anthropic_hh_rlhf:subset=hh,model=cohere_command-xlarge-beta,evaluator=gpt4", "anthropic_hh_rlhf:subset=hh,model=cohere_command-xlarge-beta,evaluator=mturk", "anthropic_hh_rlhf:subset=hh,model=cohere_command-xlarge-beta,evaluator=scale", "anthropic_hh_rlhf:subset=red_team,model=cohere_command-xlarge-beta,evaluator=claude", "anthropic_hh_rlhf:subset=red_team,model=cohere_command-xlarge-beta,evaluator=gpt4", "anthropic_hh_rlhf:subset=red_team,model=cohere_command-xlarge-beta,evaluator=mturk", "anthropic_hh_rlhf:subset=red_team,model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 4.214375, "description": "min=3.38, mean=4.214, max=4.92, sum=33.715 (8)", "style": {}, "markdown": false, "run_spec_names": [ "anthropic_hh_rlhf:subset=hh,model=cohere_command-xlarge-beta,evaluator=claude", "anthropic_hh_rlhf:subset=hh,model=cohere_command-xlarge-beta,evaluator=gpt4", "anthropic_hh_rlhf:subset=hh,model=cohere_command-xlarge-beta,evaluator=mturk", "anthropic_hh_rlhf:subset=hh,model=cohere_command-xlarge-beta,evaluator=scale", "anthropic_hh_rlhf:subset=red_team,model=cohere_command-xlarge-beta,evaluator=claude", "anthropic_hh_rlhf:subset=red_team,model=cohere_command-xlarge-beta,evaluator=gpt4", "anthropic_hh_rlhf:subset=red_team,model=cohere_command-xlarge-beta,evaluator=mturk", "anthropic_hh_rlhf:subset=red_team,model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 3.9087500000000004, "description": "min=3.68, mean=3.909, max=4.2, sum=15.635 (4)", "style": {}, "markdown": false, "run_spec_names": [ "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=cohere_command-xlarge-beta,evaluator=claude", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=cohere_command-xlarge-beta,evaluator=gpt4", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=cohere_command-xlarge-beta,evaluator=mturk", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 4.8149999999999995, "description": "min=4.685, mean=4.815, max=4.9, sum=19.26 (4)", "style": {}, "markdown": false, "run_spec_names": [ "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=cohere_command-xlarge-beta,evaluator=claude", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=cohere_command-xlarge-beta,evaluator=gpt4", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=cohere_command-xlarge-beta,evaluator=mturk", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 3.9925, "description": "min=3.75, mean=3.993, max=4.1, sum=15.97 (4)", "style": {}, "markdown": false, "run_spec_names": [ "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=cohere_command-xlarge-beta,evaluator=claude", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=cohere_command-xlarge-beta,evaluator=gpt4", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=cohere_command-xlarge-beta,evaluator=mturk", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 4.01875, "description": "min=3.51, mean=4.019, max=4.53, sum=16.075 (4)", "style": {}, "markdown": false, "run_spec_names": [ "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=cohere_command-xlarge-beta,evaluator=claude", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=cohere_command-xlarge-beta,evaluator=gpt4", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=cohere_command-xlarge-beta,evaluator=mturk", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 4.987500000000001, "description": "min=4.98, mean=4.988, max=5, sum=19.95 (4)", "style": {}, "markdown": false, "run_spec_names": [ "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=cohere_command-xlarge-beta,evaluator=claude", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=cohere_command-xlarge-beta,evaluator=gpt4", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=cohere_command-xlarge-beta,evaluator=mturk", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 3.6941174984845424, "description": "min=3.214, mean=3.694, max=4.117, sum=14.776 (4)", "style": {}, "markdown": false, "run_spec_names": [ "koala:model=cohere_command-xlarge-beta,evaluator=claude", "koala:model=cohere_command-xlarge-beta,evaluator=gpt4", "koala:model=cohere_command-xlarge-beta,evaluator=mturk", "koala:model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 4.695443523944231, "description": "min=4.629, mean=4.695, max=4.755, sum=18.782 (4)", "style": {}, "markdown": false, "run_spec_names": [ "koala:model=cohere_command-xlarge-beta,evaluator=claude", "koala:model=cohere_command-xlarge-beta,evaluator=gpt4", "koala:model=cohere_command-xlarge-beta,evaluator=mturk", "koala:model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 3.7327237825823394, "description": "min=3.235, mean=3.733, max=4.02, sum=14.931 (4)", "style": {}, "markdown": false, "run_spec_names": [ "koala:model=cohere_command-xlarge-beta,evaluator=claude", "koala:model=cohere_command-xlarge-beta,evaluator=gpt4", "koala:model=cohere_command-xlarge-beta,evaluator=mturk", "koala:model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 3.71854162457062, "description": "min=3.173, mean=3.719, max=4.26, sum=14.874 (4)", "style": {}, "markdown": false, "run_spec_names": [ "koala:model=cohere_command-xlarge-beta,evaluator=claude", "koala:model=cohere_command-xlarge-beta,evaluator=gpt4", "koala:model=cohere_command-xlarge-beta,evaluator=mturk", "koala:model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 4.968604768640129, "description": "min=4.936, mean=4.969, max=5, sum=19.874 (4)", "style": {}, "markdown": false, "run_spec_names": [ "koala:model=cohere_command-xlarge-beta,evaluator=claude", "koala:model=cohere_command-xlarge-beta,evaluator=gpt4", "koala:model=cohere_command-xlarge-beta,evaluator=mturk", "koala:model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 3.5934469696969695, "description": "min=3.3, mean=3.593, max=4.145, sum=14.374 (4)", "style": {}, "markdown": false, "run_spec_names": [ "open_assistant:language=en,model=cohere_command-xlarge-beta,evaluator=claude", "open_assistant:language=en,model=cohere_command-xlarge-beta,evaluator=gpt4", "open_assistant:language=en,model=cohere_command-xlarge-beta,evaluator=mturk", "open_assistant:language=en,model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 4.684040404040404, "description": "min=4.616, mean=4.684, max=4.74, sum=18.736 (4)", "style": {}, "markdown": false, "run_spec_names": [ "open_assistant:language=en,model=cohere_command-xlarge-beta,evaluator=claude", "open_assistant:language=en,model=cohere_command-xlarge-beta,evaluator=gpt4", "open_assistant:language=en,model=cohere_command-xlarge-beta,evaluator=mturk", "open_assistant:language=en,model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 3.672449494949495, "description": "min=3.28, mean=3.672, max=4.07, sum=14.69 (4)", "style": {}, "markdown": false, "run_spec_names": [ "open_assistant:language=en,model=cohere_command-xlarge-beta,evaluator=claude", "open_assistant:language=en,model=cohere_command-xlarge-beta,evaluator=gpt4", "open_assistant:language=en,model=cohere_command-xlarge-beta,evaluator=mturk", "open_assistant:language=en,model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 3.917121212121212, "description": "min=3.33, mean=3.917, max=4.348, sum=15.668 (4)", "style": {}, "markdown": false, "run_spec_names": [ "open_assistant:language=en,model=cohere_command-xlarge-beta,evaluator=claude", "open_assistant:language=en,model=cohere_command-xlarge-beta,evaluator=gpt4", "open_assistant:language=en,model=cohere_command-xlarge-beta,evaluator=mturk", "open_assistant:language=en,model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 4.967386363636363, "description": "min=4.955, mean=4.967, max=5, sum=19.87 (4)", "style": {}, "markdown": false, "run_spec_names": [ "open_assistant:language=en,model=cohere_command-xlarge-beta,evaluator=claude", "open_assistant:language=en,model=cohere_command-xlarge-beta,evaluator=gpt4", "open_assistant:language=en,model=cohere_command-xlarge-beta,evaluator=mturk", "open_assistant:language=en,model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 3.6626388888888886, "description": "min=3.51, mean=3.663, max=3.815, sum=14.651 (4)", "style": {}, "markdown": false, "run_spec_names": [ "self_instruct:model=cohere_command-xlarge-beta,evaluator=claude", "self_instruct:model=cohere_command-xlarge-beta,evaluator=gpt4", "self_instruct:model=cohere_command-xlarge-beta,evaluator=mturk", "self_instruct:model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 4.722878787878788, "description": "min=4.54, mean=4.723, max=4.86, sum=18.892 (4)", "style": {}, "markdown": false, "run_spec_names": [ "self_instruct:model=cohere_command-xlarge-beta,evaluator=claude", "self_instruct:model=cohere_command-xlarge-beta,evaluator=gpt4", "self_instruct:model=cohere_command-xlarge-beta,evaluator=mturk", "self_instruct:model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 3.694116161616162, "description": "min=3.39, mean=3.694, max=4.01, sum=14.776 (4)", "style": {}, "markdown": false, "run_spec_names": [ "self_instruct:model=cohere_command-xlarge-beta,evaluator=claude", "self_instruct:model=cohere_command-xlarge-beta,evaluator=gpt4", "self_instruct:model=cohere_command-xlarge-beta,evaluator=mturk", "self_instruct:model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 3.9858712121212116, "description": "min=3.6, mean=3.986, max=4.348, sum=15.943 (4)", "style": {}, "markdown": false, "run_spec_names": [ "self_instruct:model=cohere_command-xlarge-beta,evaluator=claude", "self_instruct:model=cohere_command-xlarge-beta,evaluator=gpt4", "self_instruct:model=cohere_command-xlarge-beta,evaluator=mturk", "self_instruct:model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 4.971174242424242, "description": "min=4.955, mean=4.971, max=5, sum=19.885 (4)", "style": {}, "markdown": false, "run_spec_names": [ "self_instruct:model=cohere_command-xlarge-beta,evaluator=claude", "self_instruct:model=cohere_command-xlarge-beta,evaluator=gpt4", "self_instruct:model=cohere_command-xlarge-beta,evaluator=mturk", "self_instruct:model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 3.9171875, "description": "min=3.625, mean=3.917, max=4.444, sum=15.669 (4)", "style": {}, "markdown": false, "run_spec_names": [ "vicuna:category=all,model=cohere_command-xlarge-beta,evaluator=claude", "vicuna:category=all,model=cohere_command-xlarge-beta,evaluator=gpt4", "vicuna:category=all,model=cohere_command-xlarge-beta,evaluator=mturk", "vicuna:category=all,model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 4.86875, "description": "min=4.831, mean=4.869, max=4.9, sum=19.475 (4)", "style": {}, "markdown": false, "run_spec_names": [ "vicuna:category=all,model=cohere_command-xlarge-beta,evaluator=claude", "vicuna:category=all,model=cohere_command-xlarge-beta,evaluator=gpt4", "vicuna:category=all,model=cohere_command-xlarge-beta,evaluator=mturk", "vicuna:category=all,model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 4.0484374999999995, "description": "min=3.694, mean=4.048, max=4.45, sum=16.194 (4)", "style": {}, "markdown": false, "run_spec_names": [ "vicuna:category=all,model=cohere_command-xlarge-beta,evaluator=claude", "vicuna:category=all,model=cohere_command-xlarge-beta,evaluator=gpt4", "vicuna:category=all,model=cohere_command-xlarge-beta,evaluator=mturk", "vicuna:category=all,model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 4.089062500000001, "description": "min=3.612, mean=4.089, max=4.562, sum=16.356 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "vicuna:category=all,model=cohere_command-xlarge-beta,evaluator=claude", "vicuna:category=all,model=cohere_command-xlarge-beta,evaluator=gpt4", "vicuna:category=all,model=cohere_command-xlarge-beta,evaluator=mturk", "vicuna:category=all,model=cohere_command-xlarge-beta,evaluator=scale" ] }, { "value": 4.9953125, "description": "min=4.981, mean=4.995, max=5, sum=19.981 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "vicuna:category=all,model=cohere_command-xlarge-beta,evaluator=claude", "vicuna:category=all,model=cohere_command-xlarge-beta,evaluator=gpt4", "vicuna:category=all,model=cohere_command-xlarge-beta,evaluator=mturk", "vicuna:category=all,model=cohere_command-xlarge-beta,evaluator=scale" ] } ], [ { "value": "GPT-3.5 Turbo (0613)", "description": "", "markdown": false }, { "value": 0.6888888888888889, "style": { "font-weight": "bold" }, "markdown": false }, { "value": 3.6250000000000004, "description": "min=2.89, mean=3.625, max=4.6, sum=29.0 (8)", "style": {}, "markdown": false, "run_spec_names": [ "anthropic_hh_rlhf:subset=hh,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-3.5-turbo-0613,evaluator=scale", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.97375, "description": "min=4.915, mean=4.974, max=5, sum=39.79 (8)", "style": {}, "markdown": false, "run_spec_names": [ "anthropic_hh_rlhf:subset=hh,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-3.5-turbo-0613,evaluator=scale", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 3.9074999999999998, "description": "min=2.755, mean=3.907, max=4.74, sum=31.26 (8)", "style": {}, "markdown": false, "run_spec_names": [ "anthropic_hh_rlhf:subset=hh,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-3.5-turbo-0613,evaluator=scale", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.19125, "description": "min=3.79, mean=4.191, max=4.67, sum=33.53 (8)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "anthropic_hh_rlhf:subset=hh,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-3.5-turbo-0613,evaluator=scale", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.9643749999999995, "description": "min=4.915, mean=4.964, max=5, sum=39.715 (8)", "style": {}, "markdown": false, "run_spec_names": [ "anthropic_hh_rlhf:subset=hh,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-3.5-turbo-0613,evaluator=scale", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.3129605263157895, "description": "min=3.98, mean=4.313, max=4.737, sum=17.252 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.894868421052632, "description": "min=4.72, mean=4.895, max=4.99, sum=19.579 (4)", "style": {}, "markdown": false, "run_spec_names": [ "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.773684210526316, "description": "min=4.64, mean=4.774, max=4.92, sum=19.095 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.113223684210526, "description": "min=3.75, mean=4.113, max=4.758, sum=16.453 (4)", "style": {}, "markdown": false, "run_spec_names": [ "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.986184210526316, "description": "min=4.95, mean=4.986, max=5, sum=19.945 (4)", "style": {}, "markdown": false, "run_spec_names": [ "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.164678769566079, "description": "min=3.878, mean=4.165, max=4.427, sum=16.659 (4)", "style": {}, "markdown": false, "run_spec_names": [ "koala:model=openai_gpt-3.5-turbo-0613,evaluator=claude", "koala:model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "koala:model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "koala:model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.903453784426393, "description": "min=4.775, mean=4.903, max=5, sum=19.614 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "koala:model=openai_gpt-3.5-turbo-0613,evaluator=claude", "koala:model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "koala:model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "koala:model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.568819348127601, "description": "min=4.449, mean=4.569, max=4.837, sum=18.275 (4)", "style": {}, "markdown": false, "run_spec_names": [ "koala:model=openai_gpt-3.5-turbo-0613,evaluator=claude", "koala:model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "koala:model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "koala:model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 3.9746772835347732, "description": "min=3.704, mean=3.975, max=4.223, sum=15.899 (4)", "style": {}, "markdown": false, "run_spec_names": [ "koala:model=openai_gpt-3.5-turbo-0613,evaluator=claude", "koala:model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "koala:model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "koala:model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.987383346542501, "description": "min=4.969, mean=4.987, max=5, sum=19.95 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "koala:model=openai_gpt-3.5-turbo-0613,evaluator=claude", "koala:model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "koala:model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "koala:model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.200764790764791, "description": "min=3.93, mean=4.201, max=4.52, sum=16.803 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "open_assistant:language=en,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "open_assistant:language=en,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "open_assistant:language=en,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "open_assistant:language=en,model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.9177870542156255, "description": "min=4.747, mean=4.918, max=5, sum=19.671 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "open_assistant:language=en,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "open_assistant:language=en,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "open_assistant:language=en,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "open_assistant:language=en,model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.676910430839002, "description": "min=4.577, mean=4.677, max=4.81, sum=18.708 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "open_assistant:language=en,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "open_assistant:language=en,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "open_assistant:language=en,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "open_assistant:language=en,model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.118269944341373, "description": "min=3.81, mean=4.118, max=4.495, sum=16.473 (4)", "style": {}, "markdown": false, "run_spec_names": [ "open_assistant:language=en,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "open_assistant:language=en,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "open_assistant:language=en,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "open_assistant:language=en,model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.98739898989899, "description": "min=4.96, mean=4.987, max=5, sum=19.95 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "open_assistant:language=en,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "open_assistant:language=en,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "open_assistant:language=en,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "open_assistant:language=en,model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.27, "description": "min=3.98, mean=4.27, max=4.565, sum=17.08 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "self_instruct:model=openai_gpt-3.5-turbo-0613,evaluator=claude", "self_instruct:model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "self_instruct:model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "self_instruct:model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.91, "description": "min=4.81, mean=4.91, max=4.99, sum=19.64 (4)", "style": {}, "markdown": false, "run_spec_names": [ "self_instruct:model=openai_gpt-3.5-turbo-0613,evaluator=claude", "self_instruct:model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "self_instruct:model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "self_instruct:model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.6525, "description": "min=4.45, mean=4.652, max=4.84, sum=18.61 (4)", "style": {}, "markdown": false, "run_spec_names": [ "self_instruct:model=openai_gpt-3.5-turbo-0613,evaluator=claude", "self_instruct:model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "self_instruct:model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "self_instruct:model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.2525, "description": "min=3.88, mean=4.253, max=4.705, sum=17.01 (4)", "style": {}, "markdown": false, "run_spec_names": [ "self_instruct:model=openai_gpt-3.5-turbo-0613,evaluator=claude", "self_instruct:model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "self_instruct:model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "self_instruct:model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.99, "description": "min=4.97, mean=4.99, max=5, sum=19.96 (4)", "style": {}, "markdown": false, "run_spec_names": [ "self_instruct:model=openai_gpt-3.5-turbo-0613,evaluator=claude", "self_instruct:model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "self_instruct:model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "self_instruct:model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.3265625000000005, "description": "min=4, mean=4.327, max=4.675, sum=17.306 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "vicuna:category=all,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "vicuna:category=all,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "vicuna:category=all,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "vicuna:category=all,model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.91875, "description": "min=4.775, mean=4.919, max=5, sum=19.675 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "vicuna:category=all,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "vicuna:category=all,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "vicuna:category=all,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "vicuna:category=all,model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.8671875, "description": "min=4.75, mean=4.867, max=5, sum=19.469 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "vicuna:category=all,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "vicuna:category=all,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "vicuna:category=all,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "vicuna:category=all,model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 3.8828125, "description": "min=3.587, mean=3.883, max=4.275, sum=15.531 (4)", "style": {}, "markdown": false, "run_spec_names": [ "vicuna:category=all,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "vicuna:category=all,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "vicuna:category=all,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "vicuna:category=all,model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] }, { "value": 4.9921875, "description": "min=4.975, mean=4.992, max=5, sum=19.969 (4)", "style": {}, "markdown": false, "run_spec_names": [ "vicuna:category=all,model=openai_gpt-3.5-turbo-0613,evaluator=claude", "vicuna:category=all,model=openai_gpt-3.5-turbo-0613,evaluator=gpt4", "vicuna:category=all,model=openai_gpt-3.5-turbo-0613,evaluator=mturk", "vicuna:category=all,model=openai_gpt-3.5-turbo-0613,evaluator=scale" ] } ], [ { "value": "GPT-4 (0314)", "description": "", "markdown": false }, { "value": 0.611111111111111, "markdown": false }, { "value": 3.8581250000000007, "description": "min=3.01, mean=3.858, max=4.62, sum=30.865 (8)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "anthropic_hh_rlhf:subset=hh,model=openai_gpt-4-0314,evaluator=claude", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-4-0314,evaluator=gpt4", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-4-0314,evaluator=mturk", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-4-0314,evaluator=scale", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-4-0314,evaluator=claude", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-4-0314,evaluator=gpt4", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-4-0314,evaluator=mturk", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.979375, "description": "min=4.925, mean=4.979, max=5, sum=39.835 (8)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "anthropic_hh_rlhf:subset=hh,model=openai_gpt-4-0314,evaluator=claude", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-4-0314,evaluator=gpt4", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-4-0314,evaluator=mturk", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-4-0314,evaluator=scale", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-4-0314,evaluator=claude", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-4-0314,evaluator=gpt4", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-4-0314,evaluator=mturk", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.297499999999999, "description": "min=3.39, mean=4.297, max=4.79, sum=34.38 (8)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "anthropic_hh_rlhf:subset=hh,model=openai_gpt-4-0314,evaluator=claude", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-4-0314,evaluator=gpt4", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-4-0314,evaluator=mturk", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-4-0314,evaluator=scale", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-4-0314,evaluator=claude", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-4-0314,evaluator=gpt4", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-4-0314,evaluator=mturk", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.176875, "description": "min=3.8, mean=4.177, max=4.725, sum=33.415 (8)", "style": {}, "markdown": false, "run_spec_names": [ "anthropic_hh_rlhf:subset=hh,model=openai_gpt-4-0314,evaluator=claude", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-4-0314,evaluator=gpt4", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-4-0314,evaluator=mturk", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-4-0314,evaluator=scale", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-4-0314,evaluator=claude", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-4-0314,evaluator=gpt4", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-4-0314,evaluator=mturk", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.93375, "description": "min=4.83, mean=4.934, max=5, sum=39.47 (8)", "style": {}, "markdown": false, "run_spec_names": [ "anthropic_hh_rlhf:subset=hh,model=openai_gpt-4-0314,evaluator=claude", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-4-0314,evaluator=gpt4", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-4-0314,evaluator=mturk", "anthropic_hh_rlhf:subset=hh,model=openai_gpt-4-0314,evaluator=scale", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-4-0314,evaluator=claude", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-4-0314,evaluator=gpt4", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-4-0314,evaluator=mturk", "anthropic_hh_rlhf:subset=red_team,model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.263472222222222, "description": "min=4, mean=4.263, max=4.689, sum=17.054 (4)", "style": {}, "markdown": false, "run_spec_names": [ "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-4-0314,evaluator=claude", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-4-0314,evaluator=gpt4", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-4-0314,evaluator=mturk", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.8886111111111115, "description": "min=4.7, mean=4.889, max=5, sum=19.554 (4)", "style": {}, "markdown": false, "run_spec_names": [ "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-4-0314,evaluator=claude", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-4-0314,evaluator=gpt4", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-4-0314,evaluator=mturk", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.6490277777777775, "description": "min=4.395, mean=4.649, max=4.91, sum=18.596 (4)", "style": {}, "markdown": false, "run_spec_names": [ "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-4-0314,evaluator=claude", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-4-0314,evaluator=gpt4", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-4-0314,evaluator=mturk", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.118333333333333, "description": "min=3.69, mean=4.118, max=4.783, sum=16.473 (4)", "style": {}, "markdown": false, "run_spec_names": [ "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-4-0314,evaluator=claude", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-4-0314,evaluator=gpt4", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-4-0314,evaluator=mturk", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.973472222222222, "description": "min=4.915, mean=4.973, max=5, sum=19.894 (4)", "style": {}, "markdown": false, "run_spec_names": [ "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-4-0314,evaluator=claude", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-4-0314,evaluator=gpt4", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-4-0314,evaluator=mturk", "grammar:path=src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml,tags=,model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.223441604364519, "description": "min=3.939, mean=4.223, max=4.546, sum=16.894 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "koala:model=openai_gpt-4-0314,evaluator=claude", "koala:model=openai_gpt-4-0314,evaluator=gpt4", "koala:model=openai_gpt-4-0314,evaluator=mturk", "koala:model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.885330369771671, "description": "min=4.765, mean=4.885, max=4.99, sum=19.541 (4)", "style": {}, "markdown": false, "run_spec_names": [ "koala:model=openai_gpt-4-0314,evaluator=claude", "koala:model=openai_gpt-4-0314,evaluator=gpt4", "koala:model=openai_gpt-4-0314,evaluator=mturk", "koala:model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.625366235603153, "description": "min=4.551, mean=4.625, max=4.745, sum=18.501 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "koala:model=openai_gpt-4-0314,evaluator=claude", "koala:model=openai_gpt-4-0314,evaluator=gpt4", "koala:model=openai_gpt-4-0314,evaluator=mturk", "koala:model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.044807031723581, "description": "min=3.776, mean=4.045, max=4.312, sum=16.179 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "koala:model=openai_gpt-4-0314,evaluator=claude", "koala:model=openai_gpt-4-0314,evaluator=gpt4", "koala:model=openai_gpt-4-0314,evaluator=mturk", "koala:model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.965788543140029, "description": "min=4.913, mean=4.966, max=5, sum=19.863 (4)", "style": {}, "markdown": false, "run_spec_names": [ "koala:model=openai_gpt-4-0314,evaluator=claude", "koala:model=openai_gpt-4-0314,evaluator=gpt4", "koala:model=openai_gpt-4-0314,evaluator=mturk", "koala:model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.190561224489795, "description": "min=3.92, mean=4.191, max=4.54, sum=16.762 (4)", "style": {}, "markdown": false, "run_spec_names": [ "open_assistant:language=en,model=openai_gpt-4-0314,evaluator=claude", "open_assistant:language=en,model=openai_gpt-4-0314,evaluator=gpt4", "open_assistant:language=en,model=openai_gpt-4-0314,evaluator=mturk", "open_assistant:language=en,model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.905739795918368, "description": "min=4.785, mean=4.906, max=4.98, sum=19.623 (4)", "style": {}, "markdown": false, "run_spec_names": [ "open_assistant:language=en,model=openai_gpt-4-0314,evaluator=claude", "open_assistant:language=en,model=openai_gpt-4-0314,evaluator=gpt4", "open_assistant:language=en,model=openai_gpt-4-0314,evaluator=mturk", "open_assistant:language=en,model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.669030612244898, "description": "min=4.556, mean=4.669, max=4.8, sum=18.676 (4)", "style": {}, "markdown": false, "run_spec_names": [ "open_assistant:language=en,model=openai_gpt-4-0314,evaluator=claude", "open_assistant:language=en,model=openai_gpt-4-0314,evaluator=gpt4", "open_assistant:language=en,model=openai_gpt-4-0314,evaluator=mturk", "open_assistant:language=en,model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.210561224489796, "description": "min=3.86, mean=4.211, max=4.612, sum=16.842 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "open_assistant:language=en,model=openai_gpt-4-0314,evaluator=claude", "open_assistant:language=en,model=openai_gpt-4-0314,evaluator=gpt4", "open_assistant:language=en,model=openai_gpt-4-0314,evaluator=mturk", "open_assistant:language=en,model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.9862244897959185, "description": "min=4.97, mean=4.986, max=5, sum=19.945 (4)", "style": {}, "markdown": false, "run_spec_names": [ "open_assistant:language=en,model=openai_gpt-4-0314,evaluator=claude", "open_assistant:language=en,model=openai_gpt-4-0314,evaluator=gpt4", "open_assistant:language=en,model=openai_gpt-4-0314,evaluator=mturk", "open_assistant:language=en,model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.2526785714285715, "description": "min=3.97, mean=4.253, max=4.536, sum=17.011 (4)", "style": {}, "markdown": false, "run_spec_names": [ "self_instruct:model=openai_gpt-4-0314,evaluator=claude", "self_instruct:model=openai_gpt-4-0314,evaluator=gpt4", "self_instruct:model=openai_gpt-4-0314,evaluator=mturk", "self_instruct:model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.91579081632653, "description": "min=4.765, mean=4.916, max=5, sum=19.663 (4)", "style": {}, "markdown": false, "run_spec_names": [ "self_instruct:model=openai_gpt-4-0314,evaluator=claude", "self_instruct:model=openai_gpt-4-0314,evaluator=gpt4", "self_instruct:model=openai_gpt-4-0314,evaluator=mturk", "self_instruct:model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.597857142857142, "description": "min=4.51, mean=4.598, max=4.78, sum=18.391 (4)", "style": {}, "markdown": false, "run_spec_names": [ "self_instruct:model=openai_gpt-4-0314,evaluator=claude", "self_instruct:model=openai_gpt-4-0314,evaluator=gpt4", "self_instruct:model=openai_gpt-4-0314,evaluator=mturk", "self_instruct:model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.277551020408163, "description": "min=3.97, mean=4.278, max=4.76, sum=17.11 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "self_instruct:model=openai_gpt-4-0314,evaluator=claude", "self_instruct:model=openai_gpt-4-0314,evaluator=gpt4", "self_instruct:model=openai_gpt-4-0314,evaluator=mturk", "self_instruct:model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.976198979591836, "description": "min=4.945, mean=4.976, max=5, sum=19.905 (4)", "style": {}, "markdown": false, "run_spec_names": [ "self_instruct:model=openai_gpt-4-0314,evaluator=claude", "self_instruct:model=openai_gpt-4-0314,evaluator=gpt4", "self_instruct:model=openai_gpt-4-0314,evaluator=mturk", "self_instruct:model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.323457278481013, "description": "min=4, mean=4.323, max=4.688, sum=17.294 (4)", "style": {}, "markdown": false, "run_spec_names": [ "vicuna:category=all,model=openai_gpt-4-0314,evaluator=claude", "vicuna:category=all,model=openai_gpt-4-0314,evaluator=gpt4", "vicuna:category=all,model=openai_gpt-4-0314,evaluator=mturk", "vicuna:category=all,model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.899485759493671, "description": "min=4.763, mean=4.899, max=5, sum=19.598 (4)", "style": {}, "markdown": false, "run_spec_names": [ "vicuna:category=all,model=openai_gpt-4-0314,evaluator=claude", "vicuna:category=all,model=openai_gpt-4-0314,evaluator=gpt4", "vicuna:category=all,model=openai_gpt-4-0314,evaluator=mturk", "vicuna:category=all,model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.8208662974683545, "description": "min=4.677, mean=4.821, max=4.938, sum=19.283 (4)", "style": {}, "markdown": false, "run_spec_names": [ "vicuna:category=all,model=openai_gpt-4-0314,evaluator=claude", "vicuna:category=all,model=openai_gpt-4-0314,evaluator=gpt4", "vicuna:category=all,model=openai_gpt-4-0314,evaluator=mturk", "vicuna:category=all,model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 3.879212816455696, "description": "min=3.587, mean=3.879, max=4.348, sum=15.517 (4)", "style": {}, "markdown": false, "run_spec_names": [ "vicuna:category=all,model=openai_gpt-4-0314,evaluator=claude", "vicuna:category=all,model=openai_gpt-4-0314,evaluator=gpt4", "vicuna:category=all,model=openai_gpt-4-0314,evaluator=mturk", "vicuna:category=all,model=openai_gpt-4-0314,evaluator=scale" ] }, { "value": 4.9953125, "description": "min=4.981, mean=4.995, max=5, sum=19.981 (4)", "style": { "font-weight": "bold" }, "markdown": false, "run_spec_names": [ "vicuna:category=all,model=openai_gpt-4-0314,evaluator=claude", "vicuna:category=all,model=openai_gpt-4-0314,evaluator=gpt4", "vicuna:category=all,model=openai_gpt-4-0314,evaluator=mturk", "vicuna:category=all,model=openai_gpt-4-0314,evaluator=scale" ] } ] ], "links": [ { "text": "LaTeX", "href": "benchmark_output/releases/instruction_following/groups/latex/instruction_following_instruction_following_metrics.tex" }, { "text": "JSON", "href": "benchmark_output/releases/instruction_following/groups/json/instruction_following_instruction_following_metrics.json" } ], "name": "instruction_following_metrics" } ]