| { | |
| "results": { | |
| "arc_challenge": { | |
| "alias": "arc_challenge", | |
| "acc,none": 0.39078498293515357, | |
| "acc_stderr,none": 0.014258563880513775, | |
| "acc_norm,none": 0.38993174061433444, | |
| "acc_norm_stderr,none": 0.01425295984889289 | |
| }, | |
| "mmlu": { | |
| "acc,none": 0.3787921948440393, | |
| "acc_stderr,none": 0.003987973543355471, | |
| "alias": "mmlu" | |
| }, | |
| "mmlu_humanities": { | |
| "acc,none": 0.318384697130712, | |
| "acc_stderr,none": 0.006638395002400861, | |
| "alias": " - humanities" | |
| }, | |
| "mmlu_formal_logic": { | |
| "alias": " - formal_logic", | |
| "acc,none": 0.3492063492063492, | |
| "acc_stderr,none": 0.04263906892795132 | |
| }, | |
| "mmlu_high_school_european_history": { | |
| "alias": " - high_school_european_history", | |
| "acc,none": 0.3090909090909091, | |
| "acc_stderr,none": 0.03608541011573967 | |
| }, | |
| "mmlu_high_school_us_history": { | |
| "alias": " - high_school_us_history", | |
| "acc,none": 0.28921568627450983, | |
| "acc_stderr,none": 0.03182231867647554 | |
| }, | |
| "mmlu_high_school_world_history": { | |
| "alias": " - high_school_world_history", | |
| "acc,none": 0.3080168776371308, | |
| "acc_stderr,none": 0.030052389335605705 | |
| }, | |
| "mmlu_international_law": { | |
| "alias": " - international_law", | |
| "acc,none": 0.48760330578512395, | |
| "acc_stderr,none": 0.04562951548180765 | |
| }, | |
| "mmlu_jurisprudence": { | |
| "alias": " - jurisprudence", | |
| "acc,none": 0.3333333333333333, | |
| "acc_stderr,none": 0.04557239513497751 | |
| }, | |
| "mmlu_logical_fallacies": { | |
| "alias": " - logical_fallacies", | |
| "acc,none": 0.49079754601226994, | |
| "acc_stderr,none": 0.03927705600787443 | |
| }, | |
| "mmlu_moral_disputes": { | |
| "alias": " - moral_disputes", | |
| "acc,none": 0.43641618497109824, | |
| "acc_stderr,none": 0.026700545424943677 | |
| }, | |
| "mmlu_moral_scenarios": { | |
| "alias": " - moral_scenarios", | |
| "acc,none": 0.16089385474860335, | |
| "acc_stderr,none": 0.012288798406607286 | |
| }, | |
| "mmlu_philosophy": { | |
| "alias": " - philosophy", | |
| "acc,none": 0.3890675241157556, | |
| "acc_stderr,none": 0.027690337536485376 | |
| }, | |
| "mmlu_prehistory": { | |
| "alias": " - prehistory", | |
| "acc,none": 0.39197530864197533, | |
| "acc_stderr,none": 0.027163686038271226 | |
| }, | |
| "mmlu_professional_law": { | |
| "alias": " - professional_law", | |
| "acc,none": 0.29986962190352023, | |
| "acc_stderr,none": 0.011702660860193986 | |
| }, | |
| "mmlu_world_religions": { | |
| "alias": " - world_religions", | |
| "acc,none": 0.543859649122807, | |
| "acc_stderr,none": 0.03820042586602966 | |
| }, | |
| "mmlu_other": { | |
| "acc,none": 0.44480205986482135, | |
| "acc_stderr,none": 0.00866316828289633, | |
| "alias": " - other" | |
| }, | |
| "mmlu_business_ethics": { | |
| "alias": " - business_ethics", | |
| "acc,none": 0.38, | |
| "acc_stderr,none": 0.04878317312145633 | |
| }, | |
| "mmlu_clinical_knowledge": { | |
| "alias": " - clinical_knowledge", | |
| "acc,none": 0.35471698113207545, | |
| "acc_stderr,none": 0.029445175328199593 | |
| }, | |
| "mmlu_college_medicine": { | |
| "alias": " - college_medicine", | |
| "acc,none": 0.3699421965317919, | |
| "acc_stderr,none": 0.03681229633394319 | |
| }, | |
| "mmlu_global_facts": { | |
| "alias": " - global_facts", | |
| "acc,none": 0.34, | |
| "acc_stderr,none": 0.04760952285695236 | |
| }, | |
| "mmlu_human_aging": { | |
| "alias": " - human_aging", | |
| "acc,none": 0.3183856502242152, | |
| "acc_stderr,none": 0.03126580522513713 | |
| }, | |
| "mmlu_management": { | |
| "alias": " - management", | |
| "acc,none": 0.6407766990291263, | |
| "acc_stderr,none": 0.04750458399041697 | |
| }, | |
| "mmlu_marketing": { | |
| "alias": " - marketing", | |
| "acc,none": 0.37606837606837606, | |
| "acc_stderr,none": 0.03173393632969481 | |
| }, | |
| "mmlu_medical_genetics": { | |
| "alias": " - medical_genetics", | |
| "acc,none": 0.4, | |
| "acc_stderr,none": 0.049236596391733084 | |
| }, | |
| "mmlu_miscellaneous": { | |
| "alias": " - miscellaneous", | |
| "acc,none": 0.6194125159642401, | |
| "acc_stderr,none": 0.017362564126075418 | |
| }, | |
| "mmlu_nutrition": { | |
| "alias": " - nutrition", | |
| "acc,none": 0.4117647058823529, | |
| "acc_stderr,none": 0.028180596328259283 | |
| }, | |
| "mmlu_professional_accounting": { | |
| "alias": " - professional_accounting", | |
| "acc,none": 0.3333333333333333, | |
| "acc_stderr,none": 0.028121636040639893 | |
| }, | |
| "mmlu_professional_medicine": { | |
| "alias": " - professional_medicine", | |
| "acc,none": 0.49264705882352944, | |
| "acc_stderr,none": 0.030369552523902173 | |
| }, | |
| "mmlu_virology": { | |
| "alias": " - virology", | |
| "acc,none": 0.2891566265060241, | |
| "acc_stderr,none": 0.03529486801511115 | |
| }, | |
| "mmlu_social_sciences": { | |
| "acc,none": 0.45433864153396164, | |
| "acc_stderr,none": 0.00885562286685913, | |
| "alias": " - social sciences" | |
| }, | |
| "mmlu_econometrics": { | |
| "alias": " - econometrics", | |
| "acc,none": 0.34210526315789475, | |
| "acc_stderr,none": 0.04462917535336937 | |
| }, | |
| "mmlu_high_school_geography": { | |
| "alias": " - high_school_geography", | |
| "acc,none": 0.3838383838383838, | |
| "acc_stderr,none": 0.034648816750163375 | |
| }, | |
| "mmlu_high_school_government_and_politics": { | |
| "alias": " - high_school_government_and_politics", | |
| "acc,none": 0.48704663212435234, | |
| "acc_stderr,none": 0.0360722806104775 | |
| }, | |
| "mmlu_high_school_macroeconomics": { | |
| "alias": " - high_school_macroeconomics", | |
| "acc,none": 0.3871794871794872, | |
| "acc_stderr,none": 0.02469721693087894 | |
| }, | |
| "mmlu_high_school_microeconomics": { | |
| "alias": " - high_school_microeconomics", | |
| "acc,none": 0.5126050420168067, | |
| "acc_stderr,none": 0.032468167657521745 | |
| }, | |
| "mmlu_high_school_psychology": { | |
| "alias": " - high_school_psychology", | |
| "acc,none": 0.5614678899082569, | |
| "acc_stderr,none": 0.02127471307395458 | |
| }, | |
| "mmlu_human_sexuality": { | |
| "alias": " - human_sexuality", | |
| "acc,none": 0.46564885496183206, | |
| "acc_stderr,none": 0.04374928560599736 | |
| }, | |
| "mmlu_professional_psychology": { | |
| "alias": " - professional_psychology", | |
| "acc,none": 0.3839869281045752, | |
| "acc_stderr,none": 0.019675808135281525 | |
| }, | |
| "mmlu_public_relations": { | |
| "alias": " - public_relations", | |
| "acc,none": 0.33636363636363636, | |
| "acc_stderr,none": 0.04525393596302506 | |
| }, | |
| "mmlu_security_studies": { | |
| "alias": " - security_studies", | |
| "acc,none": 0.4122448979591837, | |
| "acc_stderr,none": 0.03151236044674281 | |
| }, | |
| "mmlu_sociology": { | |
| "alias": " - sociology", | |
| "acc,none": 0.5323383084577115, | |
| "acc_stderr,none": 0.035281314729336065 | |
| }, | |
| "mmlu_us_foreign_policy": { | |
| "alias": " - us_foreign_policy", | |
| "acc,none": 0.69, | |
| "acc_stderr,none": 0.04648231987117316 | |
| }, | |
| "mmlu_stem": { | |
| "acc,none": 0.3301617507136061, | |
| "acc_stderr,none": 0.008351821869504925, | |
| "alias": " - stem" | |
| }, | |
| "mmlu_abstract_algebra": { | |
| "alias": " - abstract_algebra", | |
| "acc,none": 0.22, | |
| "acc_stderr,none": 0.04163331998932268 | |
| }, | |
| "mmlu_anatomy": { | |
| "alias": " - anatomy", | |
| "acc,none": 0.35555555555555557, | |
| "acc_stderr,none": 0.04135176749720385 | |
| }, | |
| "mmlu_astronomy": { | |
| "alias": " - astronomy", | |
| "acc,none": 0.4605263157894737, | |
| "acc_stderr,none": 0.04056242252249032 | |
| }, | |
| "mmlu_college_biology": { | |
| "alias": " - college_biology", | |
| "acc,none": 0.3472222222222222, | |
| "acc_stderr,none": 0.039812405437178615 | |
| }, | |
| "mmlu_college_chemistry": { | |
| "alias": " - college_chemistry", | |
| "acc,none": 0.25, | |
| "acc_stderr,none": 0.04351941398892446 | |
| }, | |
| "mmlu_college_computer_science": { | |
| "alias": " - college_computer_science", | |
| "acc,none": 0.32, | |
| "acc_stderr,none": 0.046882617226215034 | |
| }, | |
| "mmlu_college_mathematics": { | |
| "alias": " - college_mathematics", | |
| "acc,none": 0.31, | |
| "acc_stderr,none": 0.04648231987117316 | |
| }, | |
| "mmlu_college_physics": { | |
| "alias": " - college_physics", | |
| "acc,none": 0.35294117647058826, | |
| "acc_stderr,none": 0.047551296160629475 | |
| }, | |
| "mmlu_computer_security": { | |
| "alias": " - computer_security", | |
| "acc,none": 0.3, | |
| "acc_stderr,none": 0.046056618647183814 | |
| }, | |
| "mmlu_conceptual_physics": { | |
| "alias": " - conceptual_physics", | |
| "acc,none": 0.3148936170212766, | |
| "acc_stderr,none": 0.030363582197238174 | |
| }, | |
| "mmlu_electrical_engineering": { | |
| "alias": " - electrical_engineering", | |
| "acc,none": 0.32413793103448274, | |
| "acc_stderr,none": 0.03900432069185554 | |
| }, | |
| "mmlu_elementary_mathematics": { | |
| "alias": " - elementary_mathematics", | |
| "acc,none": 0.3544973544973545, | |
| "acc_stderr,none": 0.024636830602842 | |
| }, | |
| "mmlu_high_school_biology": { | |
| "alias": " - high_school_biology", | |
| "acc,none": 0.3548387096774194, | |
| "acc_stderr,none": 0.02721888977330876 | |
| }, | |
| "mmlu_high_school_chemistry": { | |
| "alias": " - high_school_chemistry", | |
| "acc,none": 0.33004926108374383, | |
| "acc_stderr,none": 0.03308530426228259 | |
| }, | |
| "mmlu_high_school_computer_science": { | |
| "alias": " - high_school_computer_science", | |
| "acc,none": 0.31, | |
| "acc_stderr,none": 0.04648231987117316 | |
| }, | |
| "mmlu_high_school_mathematics": { | |
| "alias": " - high_school_mathematics", | |
| "acc,none": 0.28888888888888886, | |
| "acc_stderr,none": 0.027634907264178544 | |
| }, | |
| "mmlu_high_school_physics": { | |
| "alias": " - high_school_physics", | |
| "acc,none": 0.2913907284768212, | |
| "acc_stderr,none": 0.03710185726119995 | |
| }, | |
| "mmlu_high_school_statistics": { | |
| "alias": " - high_school_statistics", | |
| "acc,none": 0.39814814814814814, | |
| "acc_stderr,none": 0.03338473403207401 | |
| }, | |
| "mmlu_machine_learning": { | |
| "alias": " - machine_learning", | |
| "acc,none": 0.23214285714285715, | |
| "acc_stderr,none": 0.04007341809755803 | |
| } | |
| }, | |
| "groups": { | |
| "mmlu": { | |
| "acc,none": 0.3787921948440393, | |
| "acc_stderr,none": 0.003987973543355471, | |
| "alias": "mmlu" | |
| }, | |
| "mmlu_humanities": { | |
| "acc,none": 0.318384697130712, | |
| "acc_stderr,none": 0.006638395002400861, | |
| "alias": " - humanities" | |
| }, | |
| "mmlu_other": { | |
| "acc,none": 0.44480205986482135, | |
| "acc_stderr,none": 0.00866316828289633, | |
| "alias": " - other" | |
| }, | |
| "mmlu_social_sciences": { | |
| "acc,none": 0.45433864153396164, | |
| "acc_stderr,none": 0.00885562286685913, | |
| "alias": " - social sciences" | |
| }, | |
| "mmlu_stem": { | |
| "acc,none": 0.3301617507136061, | |
| "acc_stderr,none": 0.008351821869504925, | |
| "alias": " - stem" | |
| } | |
| }, | |
| "group_subtasks": { | |
| "arc_challenge": [], | |
| "mmlu_humanities": [ | |
| "mmlu_formal_logic", | |
| "mmlu_prehistory", | |
| "mmlu_world_religions", | |
| "mmlu_philosophy", | |
| "mmlu_high_school_world_history", | |
| "mmlu_professional_law", | |
| "mmlu_high_school_us_history", | |
| "mmlu_logical_fallacies", | |
| "mmlu_international_law", | |
| "mmlu_high_school_european_history", | |
| "mmlu_moral_disputes", | |
| "mmlu_moral_scenarios", | |
| "mmlu_jurisprudence" | |
| ], | |
| "mmlu_social_sciences": [ | |
| "mmlu_public_relations", | |
| "mmlu_sociology", | |
| "mmlu_security_studies", | |
| "mmlu_high_school_government_and_politics", | |
| "mmlu_high_school_psychology", | |
| "mmlu_human_sexuality", | |
| "mmlu_us_foreign_policy", | |
| "mmlu_high_school_microeconomics", | |
| "mmlu_econometrics", | |
| "mmlu_high_school_macroeconomics", | |
| "mmlu_high_school_geography", | |
| "mmlu_professional_psychology" | |
| ], | |
| "mmlu_other": [ | |
| "mmlu_medical_genetics", | |
| "mmlu_global_facts", | |
| "mmlu_marketing", | |
| "mmlu_college_medicine", | |
| "mmlu_human_aging", | |
| "mmlu_virology", | |
| "mmlu_business_ethics", | |
| "mmlu_clinical_knowledge", | |
| "mmlu_professional_medicine", | |
| "mmlu_nutrition", | |
| "mmlu_miscellaneous", | |
| "mmlu_professional_accounting", | |
| "mmlu_management" | |
| ], | |
| "mmlu_stem": [ | |
| "mmlu_conceptual_physics", | |
| "mmlu_high_school_chemistry", | |
| "mmlu_college_biology", | |
| "mmlu_college_chemistry", | |
| "mmlu_machine_learning", | |
| "mmlu_elementary_mathematics", | |
| "mmlu_abstract_algebra", | |
| "mmlu_astronomy", | |
| "mmlu_high_school_statistics", | |
| "mmlu_anatomy", | |
| "mmlu_college_mathematics", | |
| "mmlu_computer_security", | |
| "mmlu_college_computer_science", | |
| "mmlu_electrical_engineering", | |
| "mmlu_college_physics", | |
| "mmlu_high_school_computer_science", | |
| "mmlu_high_school_physics", | |
| "mmlu_high_school_biology", | |
| "mmlu_high_school_mathematics" | |
| ], | |
| "mmlu": [ | |
| "mmlu_stem", | |
| "mmlu_other", | |
| "mmlu_social_sciences", | |
| "mmlu_humanities" | |
| ] | |
| }, | |
| "configs": { | |
| "arc_challenge": { | |
| "task": "arc_challenge", | |
| "tag": [ | |
| "ai2_arc" | |
| ], | |
| "dataset_path": "allenai/ai2_arc", | |
| "dataset_name": "ARC-Challenge", | |
| "training_split": "train", | |
| "validation_split": "validation", | |
| "test_split": "test", | |
| "doc_to_text": "Question: {{question}}\nAnswer:", | |
| "doc_to_target": "{{choices.label.index(answerKey)}}", | |
| "doc_to_choice": "{{choices.text}}", | |
| "description": "", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "acc_norm", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": true, | |
| "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_abstract_algebra": { | |
| "task": "mmlu_abstract_algebra", | |
| "task_alias": "abstract_algebra", | |
| "tag": "mmlu_stem_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "abstract_algebra", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about abstract algebra.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_anatomy": { | |
| "task": "mmlu_anatomy", | |
| "task_alias": "anatomy", | |
| "tag": "mmlu_stem_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "anatomy", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about anatomy.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_astronomy": { | |
| "task": "mmlu_astronomy", | |
| "task_alias": "astronomy", | |
| "tag": "mmlu_stem_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "astronomy", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about astronomy.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_business_ethics": { | |
| "task": "mmlu_business_ethics", | |
| "task_alias": "business_ethics", | |
| "tag": "mmlu_other_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "business_ethics", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about business ethics.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_clinical_knowledge": { | |
| "task": "mmlu_clinical_knowledge", | |
| "task_alias": "clinical_knowledge", | |
| "tag": "mmlu_other_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "clinical_knowledge", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about clinical knowledge.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_college_biology": { | |
| "task": "mmlu_college_biology", | |
| "task_alias": "college_biology", | |
| "tag": "mmlu_stem_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "college_biology", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about college biology.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_college_chemistry": { | |
| "task": "mmlu_college_chemistry", | |
| "task_alias": "college_chemistry", | |
| "tag": "mmlu_stem_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "college_chemistry", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about college chemistry.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_college_computer_science": { | |
| "task": "mmlu_college_computer_science", | |
| "task_alias": "college_computer_science", | |
| "tag": "mmlu_stem_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "college_computer_science", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about college computer science.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_college_mathematics": { | |
| "task": "mmlu_college_mathematics", | |
| "task_alias": "college_mathematics", | |
| "tag": "mmlu_stem_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "college_mathematics", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about college mathematics.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_college_medicine": { | |
| "task": "mmlu_college_medicine", | |
| "task_alias": "college_medicine", | |
| "tag": "mmlu_other_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "college_medicine", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about college medicine.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_college_physics": { | |
| "task": "mmlu_college_physics", | |
| "task_alias": "college_physics", | |
| "tag": "mmlu_stem_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "college_physics", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about college physics.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_computer_security": { | |
| "task": "mmlu_computer_security", | |
| "task_alias": "computer_security", | |
| "tag": "mmlu_stem_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "computer_security", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about computer security.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_conceptual_physics": { | |
| "task": "mmlu_conceptual_physics", | |
| "task_alias": "conceptual_physics", | |
| "tag": "mmlu_stem_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "conceptual_physics", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about conceptual physics.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_econometrics": { | |
| "task": "mmlu_econometrics", | |
| "task_alias": "econometrics", | |
| "tag": "mmlu_social_sciences_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "econometrics", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about econometrics.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_electrical_engineering": { | |
| "task": "mmlu_electrical_engineering", | |
| "task_alias": "electrical_engineering", | |
| "tag": "mmlu_stem_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "electrical_engineering", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about electrical engineering.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_elementary_mathematics": { | |
| "task": "mmlu_elementary_mathematics", | |
| "task_alias": "elementary_mathematics", | |
| "tag": "mmlu_stem_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "elementary_mathematics", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about elementary mathematics.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_formal_logic": { | |
| "task": "mmlu_formal_logic", | |
| "task_alias": "formal_logic", | |
| "tag": "mmlu_humanities_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "formal_logic", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about formal logic.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_global_facts": { | |
| "task": "mmlu_global_facts", | |
| "task_alias": "global_facts", | |
| "tag": "mmlu_other_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "global_facts", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about global facts.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_high_school_biology": { | |
| "task": "mmlu_high_school_biology", | |
| "task_alias": "high_school_biology", | |
| "tag": "mmlu_stem_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "high_school_biology", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about high school biology.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_high_school_chemistry": { | |
| "task": "mmlu_high_school_chemistry", | |
| "task_alias": "high_school_chemistry", | |
| "tag": "mmlu_stem_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "high_school_chemistry", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about high school chemistry.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_high_school_computer_science": { | |
| "task": "mmlu_high_school_computer_science", | |
| "task_alias": "high_school_computer_science", | |
| "tag": "mmlu_stem_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "high_school_computer_science", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about high school computer science.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_high_school_european_history": { | |
| "task": "mmlu_high_school_european_history", | |
| "task_alias": "high_school_european_history", | |
| "tag": "mmlu_humanities_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "high_school_european_history", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about high school european history.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_high_school_geography": { | |
| "task": "mmlu_high_school_geography", | |
| "task_alias": "high_school_geography", | |
| "tag": "mmlu_social_sciences_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "high_school_geography", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about high school geography.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_high_school_government_and_politics": { | |
| "task": "mmlu_high_school_government_and_politics", | |
| "task_alias": "high_school_government_and_politics", | |
| "tag": "mmlu_social_sciences_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "high_school_government_and_politics", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about high school government and politics.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_high_school_macroeconomics": { | |
| "task": "mmlu_high_school_macroeconomics", | |
| "task_alias": "high_school_macroeconomics", | |
| "tag": "mmlu_social_sciences_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "high_school_macroeconomics", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about high school macroeconomics.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_high_school_mathematics": { | |
| "task": "mmlu_high_school_mathematics", | |
| "task_alias": "high_school_mathematics", | |
| "tag": "mmlu_stem_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "high_school_mathematics", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about high school mathematics.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_high_school_microeconomics": { | |
| "task": "mmlu_high_school_microeconomics", | |
| "task_alias": "high_school_microeconomics", | |
| "tag": "mmlu_social_sciences_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "high_school_microeconomics", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about high school microeconomics.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_high_school_physics": { | |
| "task": "mmlu_high_school_physics", | |
| "task_alias": "high_school_physics", | |
| "tag": "mmlu_stem_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "high_school_physics", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about high school physics.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_high_school_psychology": { | |
| "task": "mmlu_high_school_psychology", | |
| "task_alias": "high_school_psychology", | |
| "tag": "mmlu_social_sciences_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "high_school_psychology", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about high school psychology.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_high_school_statistics": { | |
| "task": "mmlu_high_school_statistics", | |
| "task_alias": "high_school_statistics", | |
| "tag": "mmlu_stem_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "high_school_statistics", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about high school statistics.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_high_school_us_history": { | |
| "task": "mmlu_high_school_us_history", | |
| "task_alias": "high_school_us_history", | |
| "tag": "mmlu_humanities_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "high_school_us_history", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about high school us history.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_high_school_world_history": { | |
| "task": "mmlu_high_school_world_history", | |
| "task_alias": "high_school_world_history", | |
| "tag": "mmlu_humanities_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "high_school_world_history", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about high school world history.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_human_aging": { | |
| "task": "mmlu_human_aging", | |
| "task_alias": "human_aging", | |
| "tag": "mmlu_other_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "human_aging", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about human aging.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_human_sexuality": { | |
| "task": "mmlu_human_sexuality", | |
| "task_alias": "human_sexuality", | |
| "tag": "mmlu_social_sciences_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "human_sexuality", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about human sexuality.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_international_law": { | |
| "task": "mmlu_international_law", | |
| "task_alias": "international_law", | |
| "tag": "mmlu_humanities_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "international_law", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about international law.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_jurisprudence": { | |
| "task": "mmlu_jurisprudence", | |
| "task_alias": "jurisprudence", | |
| "tag": "mmlu_humanities_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "jurisprudence", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about jurisprudence.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_logical_fallacies": { | |
| "task": "mmlu_logical_fallacies", | |
| "task_alias": "logical_fallacies", | |
| "tag": "mmlu_humanities_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "logical_fallacies", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about logical fallacies.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_machine_learning": { | |
| "task": "mmlu_machine_learning", | |
| "task_alias": "machine_learning", | |
| "tag": "mmlu_stem_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "machine_learning", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about machine learning.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_management": { | |
| "task": "mmlu_management", | |
| "task_alias": "management", | |
| "tag": "mmlu_other_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "management", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about management.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_marketing": { | |
| "task": "mmlu_marketing", | |
| "task_alias": "marketing", | |
| "tag": "mmlu_other_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "marketing", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about marketing.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_medical_genetics": { | |
| "task": "mmlu_medical_genetics", | |
| "task_alias": "medical_genetics", | |
| "tag": "mmlu_other_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "medical_genetics", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about medical genetics.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_miscellaneous": { | |
| "task": "mmlu_miscellaneous", | |
| "task_alias": "miscellaneous", | |
| "tag": "mmlu_other_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "miscellaneous", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about miscellaneous.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_moral_disputes": { | |
| "task": "mmlu_moral_disputes", | |
| "task_alias": "moral_disputes", | |
| "tag": "mmlu_humanities_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "moral_disputes", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about moral disputes.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_moral_scenarios": { | |
| "task": "mmlu_moral_scenarios", | |
| "task_alias": "moral_scenarios", | |
| "tag": "mmlu_humanities_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "moral_scenarios", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about moral scenarios.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_nutrition": { | |
| "task": "mmlu_nutrition", | |
| "task_alias": "nutrition", | |
| "tag": "mmlu_other_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "nutrition", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about nutrition.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_philosophy": { | |
| "task": "mmlu_philosophy", | |
| "task_alias": "philosophy", | |
| "tag": "mmlu_humanities_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "philosophy", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about philosophy.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_prehistory": { | |
| "task": "mmlu_prehistory", | |
| "task_alias": "prehistory", | |
| "tag": "mmlu_humanities_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "prehistory", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about prehistory.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_professional_accounting": { | |
| "task": "mmlu_professional_accounting", | |
| "task_alias": "professional_accounting", | |
| "tag": "mmlu_other_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "professional_accounting", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about professional accounting.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_professional_law": { | |
| "task": "mmlu_professional_law", | |
| "task_alias": "professional_law", | |
| "tag": "mmlu_humanities_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "professional_law", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about professional law.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_professional_medicine": { | |
| "task": "mmlu_professional_medicine", | |
| "task_alias": "professional_medicine", | |
| "tag": "mmlu_other_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "professional_medicine", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about professional medicine.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_professional_psychology": { | |
| "task": "mmlu_professional_psychology", | |
| "task_alias": "professional_psychology", | |
| "tag": "mmlu_social_sciences_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "professional_psychology", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about professional psychology.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_public_relations": { | |
| "task": "mmlu_public_relations", | |
| "task_alias": "public_relations", | |
| "tag": "mmlu_social_sciences_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "public_relations", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about public relations.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_security_studies": { | |
| "task": "mmlu_security_studies", | |
| "task_alias": "security_studies", | |
| "tag": "mmlu_social_sciences_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "security_studies", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about security studies.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_sociology": { | |
| "task": "mmlu_sociology", | |
| "task_alias": "sociology", | |
| "tag": "mmlu_social_sciences_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "sociology", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about sociology.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_us_foreign_policy": { | |
| "task": "mmlu_us_foreign_policy", | |
| "task_alias": "us_foreign_policy", | |
| "tag": "mmlu_social_sciences_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "us_foreign_policy", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about us foreign policy.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_virology": { | |
| "task": "mmlu_virology", | |
| "task_alias": "virology", | |
| "tag": "mmlu_other_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "virology", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about virology.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "mmlu_world_religions": { | |
| "task": "mmlu_world_religions", | |
| "task_alias": "world_religions", | |
| "tag": "mmlu_humanities_tasks", | |
| "dataset_path": "hails/mmlu_no_train", | |
| "dataset_name": "world_religions", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "test_split": "test", | |
| "fewshot_split": "dev", | |
| "doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:", | |
| "doc_to_target": "answer", | |
| "doc_to_choice": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "description": "The following are multiple choice questions (with answers) about world religions.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| } | |
| }, | |
| "versions": { | |
| "arc_challenge": 1.0, | |
| "mmlu": 2, | |
| "mmlu_abstract_algebra": 1.0, | |
| "mmlu_anatomy": 1.0, | |
| "mmlu_astronomy": 1.0, | |
| "mmlu_business_ethics": 1.0, | |
| "mmlu_clinical_knowledge": 1.0, | |
| "mmlu_college_biology": 1.0, | |
| "mmlu_college_chemistry": 1.0, | |
| "mmlu_college_computer_science": 1.0, | |
| "mmlu_college_mathematics": 1.0, | |
| "mmlu_college_medicine": 1.0, | |
| "mmlu_college_physics": 1.0, | |
| "mmlu_computer_security": 1.0, | |
| "mmlu_conceptual_physics": 1.0, | |
| "mmlu_econometrics": 1.0, | |
| "mmlu_electrical_engineering": 1.0, | |
| "mmlu_elementary_mathematics": 1.0, | |
| "mmlu_formal_logic": 1.0, | |
| "mmlu_global_facts": 1.0, | |
| "mmlu_high_school_biology": 1.0, | |
| "mmlu_high_school_chemistry": 1.0, | |
| "mmlu_high_school_computer_science": 1.0, | |
| "mmlu_high_school_european_history": 1.0, | |
| "mmlu_high_school_geography": 1.0, | |
| "mmlu_high_school_government_and_politics": 1.0, | |
| "mmlu_high_school_macroeconomics": 1.0, | |
| "mmlu_high_school_mathematics": 1.0, | |
| "mmlu_high_school_microeconomics": 1.0, | |
| "mmlu_high_school_physics": 1.0, | |
| "mmlu_high_school_psychology": 1.0, | |
| "mmlu_high_school_statistics": 1.0, | |
| "mmlu_high_school_us_history": 1.0, | |
| "mmlu_high_school_world_history": 1.0, | |
| "mmlu_human_aging": 1.0, | |
| "mmlu_human_sexuality": 1.0, | |
| "mmlu_humanities": 2, | |
| "mmlu_international_law": 1.0, | |
| "mmlu_jurisprudence": 1.0, | |
| "mmlu_logical_fallacies": 1.0, | |
| "mmlu_machine_learning": 1.0, | |
| "mmlu_management": 1.0, | |
| "mmlu_marketing": 1.0, | |
| "mmlu_medical_genetics": 1.0, | |
| "mmlu_miscellaneous": 1.0, | |
| "mmlu_moral_disputes": 1.0, | |
| "mmlu_moral_scenarios": 1.0, | |
| "mmlu_nutrition": 1.0, | |
| "mmlu_other": 2, | |
| "mmlu_philosophy": 1.0, | |
| "mmlu_prehistory": 1.0, | |
| "mmlu_professional_accounting": 1.0, | |
| "mmlu_professional_law": 1.0, | |
| "mmlu_professional_medicine": 1.0, | |
| "mmlu_professional_psychology": 1.0, | |
| "mmlu_public_relations": 1.0, | |
| "mmlu_security_studies": 1.0, | |
| "mmlu_social_sciences": 2, | |
| "mmlu_sociology": 1.0, | |
| "mmlu_stem": 2, | |
| "mmlu_us_foreign_policy": 1.0, | |
| "mmlu_virology": 1.0, | |
| "mmlu_world_religions": 1.0 | |
| }, | |
| "n-shot": { | |
| "arc_challenge": 0, | |
| "mmlu_abstract_algebra": 0, | |
| "mmlu_anatomy": 0, | |
| "mmlu_astronomy": 0, | |
| "mmlu_business_ethics": 0, | |
| "mmlu_clinical_knowledge": 0, | |
| "mmlu_college_biology": 0, | |
| "mmlu_college_chemistry": 0, | |
| "mmlu_college_computer_science": 0, | |
| "mmlu_college_mathematics": 0, | |
| "mmlu_college_medicine": 0, | |
| "mmlu_college_physics": 0, | |
| "mmlu_computer_security": 0, | |
| "mmlu_conceptual_physics": 0, | |
| "mmlu_econometrics": 0, | |
| "mmlu_electrical_engineering": 0, | |
| "mmlu_elementary_mathematics": 0, | |
| "mmlu_formal_logic": 0, | |
| "mmlu_global_facts": 0, | |
| "mmlu_high_school_biology": 0, | |
| "mmlu_high_school_chemistry": 0, | |
| "mmlu_high_school_computer_science": 0, | |
| "mmlu_high_school_european_history": 0, | |
| "mmlu_high_school_geography": 0, | |
| "mmlu_high_school_government_and_politics": 0, | |
| "mmlu_high_school_macroeconomics": 0, | |
| "mmlu_high_school_mathematics": 0, | |
| "mmlu_high_school_microeconomics": 0, | |
| "mmlu_high_school_physics": 0, | |
| "mmlu_high_school_psychology": 0, | |
| "mmlu_high_school_statistics": 0, | |
| "mmlu_high_school_us_history": 0, | |
| "mmlu_high_school_world_history": 0, | |
| "mmlu_human_aging": 0, | |
| "mmlu_human_sexuality": 0, | |
| "mmlu_international_law": 0, | |
| "mmlu_jurisprudence": 0, | |
| "mmlu_logical_fallacies": 0, | |
| "mmlu_machine_learning": 0, | |
| "mmlu_management": 0, | |
| "mmlu_marketing": 0, | |
| "mmlu_medical_genetics": 0, | |
| "mmlu_miscellaneous": 0, | |
| "mmlu_moral_disputes": 0, | |
| "mmlu_moral_scenarios": 0, | |
| "mmlu_nutrition": 0, | |
| "mmlu_philosophy": 0, | |
| "mmlu_prehistory": 0, | |
| "mmlu_professional_accounting": 0, | |
| "mmlu_professional_law": 0, | |
| "mmlu_professional_medicine": 0, | |
| "mmlu_professional_psychology": 0, | |
| "mmlu_public_relations": 0, | |
| "mmlu_security_studies": 0, | |
| "mmlu_sociology": 0, | |
| "mmlu_us_foreign_policy": 0, | |
| "mmlu_virology": 0, | |
| "mmlu_world_religions": 0 | |
| }, | |
| "higher_is_better": { | |
| "arc_challenge": { | |
| "acc": true, | |
| "acc_norm": true | |
| }, | |
| "mmlu": { | |
| "acc": true | |
| }, | |
| "mmlu_abstract_algebra": { | |
| "acc": true | |
| }, | |
| "mmlu_anatomy": { | |
| "acc": true | |
| }, | |
| "mmlu_astronomy": { | |
| "acc": true | |
| }, | |
| "mmlu_business_ethics": { | |
| "acc": true | |
| }, | |
| "mmlu_clinical_knowledge": { | |
| "acc": true | |
| }, | |
| "mmlu_college_biology": { | |
| "acc": true | |
| }, | |
| "mmlu_college_chemistry": { | |
| "acc": true | |
| }, | |
| "mmlu_college_computer_science": { | |
| "acc": true | |
| }, | |
| "mmlu_college_mathematics": { | |
| "acc": true | |
| }, | |
| "mmlu_college_medicine": { | |
| "acc": true | |
| }, | |
| "mmlu_college_physics": { | |
| "acc": true | |
| }, | |
| "mmlu_computer_security": { | |
| "acc": true | |
| }, | |
| "mmlu_conceptual_physics": { | |
| "acc": true | |
| }, | |
| "mmlu_econometrics": { | |
| "acc": true | |
| }, | |
| "mmlu_electrical_engineering": { | |
| "acc": true | |
| }, | |
| "mmlu_elementary_mathematics": { | |
| "acc": true | |
| }, | |
| "mmlu_formal_logic": { | |
| "acc": true | |
| }, | |
| "mmlu_global_facts": { | |
| "acc": true | |
| }, | |
| "mmlu_high_school_biology": { | |
| "acc": true | |
| }, | |
| "mmlu_high_school_chemistry": { | |
| "acc": true | |
| }, | |
| "mmlu_high_school_computer_science": { | |
| "acc": true | |
| }, | |
| "mmlu_high_school_european_history": { | |
| "acc": true | |
| }, | |
| "mmlu_high_school_geography": { | |
| "acc": true | |
| }, | |
| "mmlu_high_school_government_and_politics": { | |
| "acc": true | |
| }, | |
| "mmlu_high_school_macroeconomics": { | |
| "acc": true | |
| }, | |
| "mmlu_high_school_mathematics": { | |
| "acc": true | |
| }, | |
| "mmlu_high_school_microeconomics": { | |
| "acc": true | |
| }, | |
| "mmlu_high_school_physics": { | |
| "acc": true | |
| }, | |
| "mmlu_high_school_psychology": { | |
| "acc": true | |
| }, | |
| "mmlu_high_school_statistics": { | |
| "acc": true | |
| }, | |
| "mmlu_high_school_us_history": { | |
| "acc": true | |
| }, | |
| "mmlu_high_school_world_history": { | |
| "acc": true | |
| }, | |
| "mmlu_human_aging": { | |
| "acc": true | |
| }, | |
| "mmlu_human_sexuality": { | |
| "acc": true | |
| }, | |
| "mmlu_humanities": { | |
| "acc": true | |
| }, | |
| "mmlu_international_law": { | |
| "acc": true | |
| }, | |
| "mmlu_jurisprudence": { | |
| "acc": true | |
| }, | |
| "mmlu_logical_fallacies": { | |
| "acc": true | |
| }, | |
| "mmlu_machine_learning": { | |
| "acc": true | |
| }, | |
| "mmlu_management": { | |
| "acc": true | |
| }, | |
| "mmlu_marketing": { | |
| "acc": true | |
| }, | |
| "mmlu_medical_genetics": { | |
| "acc": true | |
| }, | |
| "mmlu_miscellaneous": { | |
| "acc": true | |
| }, | |
| "mmlu_moral_disputes": { | |
| "acc": true | |
| }, | |
| "mmlu_moral_scenarios": { | |
| "acc": true | |
| }, | |
| "mmlu_nutrition": { | |
| "acc": true | |
| }, | |
| "mmlu_other": { | |
| "acc": true | |
| }, | |
| "mmlu_philosophy": { | |
| "acc": true | |
| }, | |
| "mmlu_prehistory": { | |
| "acc": true | |
| }, | |
| "mmlu_professional_accounting": { | |
| "acc": true | |
| }, | |
| "mmlu_professional_law": { | |
| "acc": true | |
| }, | |
| "mmlu_professional_medicine": { | |
| "acc": true | |
| }, | |
| "mmlu_professional_psychology": { | |
| "acc": true | |
| }, | |
| "mmlu_public_relations": { | |
| "acc": true | |
| }, | |
| "mmlu_security_studies": { | |
| "acc": true | |
| }, | |
| "mmlu_social_sciences": { | |
| "acc": true | |
| }, | |
| "mmlu_sociology": { | |
| "acc": true | |
| }, | |
| "mmlu_stem": { | |
| "acc": true | |
| }, | |
| "mmlu_us_foreign_policy": { | |
| "acc": true | |
| }, | |
| "mmlu_virology": { | |
| "acc": true | |
| }, | |
| "mmlu_world_religions": { | |
| "acc": true | |
| } | |
| }, | |
| "n-samples": { | |
| "mmlu_conceptual_physics": { | |
| "original": 235, | |
| "effective": 235 | |
| }, | |
| "mmlu_high_school_chemistry": { | |
| "original": 203, | |
| "effective": 203 | |
| }, | |
| "mmlu_college_biology": { | |
| "original": 144, | |
| "effective": 144 | |
| }, | |
| "mmlu_college_chemistry": { | |
| "original": 100, | |
| "effective": 100 | |
| }, | |
| "mmlu_machine_learning": { | |
| "original": 112, | |
| "effective": 112 | |
| }, | |
| "mmlu_elementary_mathematics": { | |
| "original": 378, | |
| "effective": 378 | |
| }, | |
| "mmlu_abstract_algebra": { | |
| "original": 100, | |
| "effective": 100 | |
| }, | |
| "mmlu_astronomy": { | |
| "original": 152, | |
| "effective": 152 | |
| }, | |
| "mmlu_high_school_statistics": { | |
| "original": 216, | |
| "effective": 216 | |
| }, | |
| "mmlu_anatomy": { | |
| "original": 135, | |
| "effective": 135 | |
| }, | |
| "mmlu_college_mathematics": { | |
| "original": 100, | |
| "effective": 100 | |
| }, | |
| "mmlu_computer_security": { | |
| "original": 100, | |
| "effective": 100 | |
| }, | |
| "mmlu_college_computer_science": { | |
| "original": 100, | |
| "effective": 100 | |
| }, | |
| "mmlu_electrical_engineering": { | |
| "original": 145, | |
| "effective": 145 | |
| }, | |
| "mmlu_college_physics": { | |
| "original": 102, | |
| "effective": 102 | |
| }, | |
| "mmlu_high_school_computer_science": { | |
| "original": 100, | |
| "effective": 100 | |
| }, | |
| "mmlu_high_school_physics": { | |
| "original": 151, | |
| "effective": 151 | |
| }, | |
| "mmlu_high_school_biology": { | |
| "original": 310, | |
| "effective": 310 | |
| }, | |
| "mmlu_high_school_mathematics": { | |
| "original": 270, | |
| "effective": 270 | |
| }, | |
| "mmlu_medical_genetics": { | |
| "original": 100, | |
| "effective": 100 | |
| }, | |
| "mmlu_global_facts": { | |
| "original": 100, | |
| "effective": 100 | |
| }, | |
| "mmlu_marketing": { | |
| "original": 234, | |
| "effective": 234 | |
| }, | |
| "mmlu_college_medicine": { | |
| "original": 173, | |
| "effective": 173 | |
| }, | |
| "mmlu_human_aging": { | |
| "original": 223, | |
| "effective": 223 | |
| }, | |
| "mmlu_virology": { | |
| "original": 166, | |
| "effective": 166 | |
| }, | |
| "mmlu_business_ethics": { | |
| "original": 100, | |
| "effective": 100 | |
| }, | |
| "mmlu_clinical_knowledge": { | |
| "original": 265, | |
| "effective": 265 | |
| }, | |
| "mmlu_professional_medicine": { | |
| "original": 272, | |
| "effective": 272 | |
| }, | |
| "mmlu_nutrition": { | |
| "original": 306, | |
| "effective": 306 | |
| }, | |
| "mmlu_miscellaneous": { | |
| "original": 783, | |
| "effective": 783 | |
| }, | |
| "mmlu_professional_accounting": { | |
| "original": 282, | |
| "effective": 282 | |
| }, | |
| "mmlu_management": { | |
| "original": 103, | |
| "effective": 103 | |
| }, | |
| "mmlu_public_relations": { | |
| "original": 110, | |
| "effective": 110 | |
| }, | |
| "mmlu_sociology": { | |
| "original": 201, | |
| "effective": 201 | |
| }, | |
| "mmlu_security_studies": { | |
| "original": 245, | |
| "effective": 245 | |
| }, | |
| "mmlu_high_school_government_and_politics": { | |
| "original": 193, | |
| "effective": 193 | |
| }, | |
| "mmlu_high_school_psychology": { | |
| "original": 545, | |
| "effective": 545 | |
| }, | |
| "mmlu_human_sexuality": { | |
| "original": 131, | |
| "effective": 131 | |
| }, | |
| "mmlu_us_foreign_policy": { | |
| "original": 100, | |
| "effective": 100 | |
| }, | |
| "mmlu_high_school_microeconomics": { | |
| "original": 238, | |
| "effective": 238 | |
| }, | |
| "mmlu_econometrics": { | |
| "original": 114, | |
| "effective": 114 | |
| }, | |
| "mmlu_high_school_macroeconomics": { | |
| "original": 390, | |
| "effective": 390 | |
| }, | |
| "mmlu_high_school_geography": { | |
| "original": 198, | |
| "effective": 198 | |
| }, | |
| "mmlu_professional_psychology": { | |
| "original": 612, | |
| "effective": 612 | |
| }, | |
| "mmlu_formal_logic": { | |
| "original": 126, | |
| "effective": 126 | |
| }, | |
| "mmlu_prehistory": { | |
| "original": 324, | |
| "effective": 324 | |
| }, | |
| "mmlu_world_religions": { | |
| "original": 171, | |
| "effective": 171 | |
| }, | |
| "mmlu_philosophy": { | |
| "original": 311, | |
| "effective": 311 | |
| }, | |
| "mmlu_high_school_world_history": { | |
| "original": 237, | |
| "effective": 237 | |
| }, | |
| "mmlu_professional_law": { | |
| "original": 1534, | |
| "effective": 1534 | |
| }, | |
| "mmlu_high_school_us_history": { | |
| "original": 204, | |
| "effective": 204 | |
| }, | |
| "mmlu_logical_fallacies": { | |
| "original": 163, | |
| "effective": 163 | |
| }, | |
| "mmlu_international_law": { | |
| "original": 121, | |
| "effective": 121 | |
| }, | |
| "mmlu_high_school_european_history": { | |
| "original": 165, | |
| "effective": 165 | |
| }, | |
| "mmlu_moral_disputes": { | |
| "original": 346, | |
| "effective": 346 | |
| }, | |
| "mmlu_moral_scenarios": { | |
| "original": 895, | |
| "effective": 895 | |
| }, | |
| "mmlu_jurisprudence": { | |
| "original": 108, | |
| "effective": 108 | |
| }, | |
| "arc_challenge": { | |
| "original": 1172, | |
| "effective": 1172 | |
| } | |
| }, | |
| "config": { | |
| "model": "hf", | |
| "model_args": "pretrained=DeepSeek-R1-Distill-Llama-70B-gptq-4bit,gptqmodel=True", | |
| "model_num_parameters": 2102665216, | |
| "model_dtype": "torch.float16", | |
| "model_revision": "main", | |
| "model_sha": "", | |
| "batch_size": 1, | |
| "batch_sizes": [], | |
| "device": null, | |
| "use_cache": null, | |
| "limit": null, | |
| "bootstrap_iters": 100000, | |
| "gen_kwargs": null, | |
| "random_seed": 0, | |
| "numpy_seed": 1234, | |
| "torch_seed": 1234, | |
| "fewshot_seed": 1234 | |
| }, | |
| "git_hash": "170660f", | |
| "date": 1739026269.7604005, | |
| "pretty_env_info": "PyTorch version: 2.6.0+cu124\nIs debug build: False\nCUDA used to build PyTorch: 12.4\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.1 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.39\n\nPython version: 3.10.16 (main, Dec 11 2024, 16:24:50) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-6.8.0-1021-azure-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: 12.4.99\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: GPU 0: NVIDIA A100 80GB PCIe\nNvidia driver version: 550.54.14\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 24\nOn-line CPU(s) list: 0-23\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V13 64-Core Processor\nCPU family: 25\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 24\nSocket(s): 1\nStepping: 1\nBogoMIPS: 4890.86\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves user_shstk clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 768 KiB (24 instances)\nL1i cache: 768 KiB (24 instances)\nL2 cache: 12 MiB (24 instances)\nL3 cache: 96 MiB (3 instances)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-23\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Vulnerable: Safe RET, no microcode\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.1.2\n[pip3] nvidia-cublas-cu12==12.4.5.8\n[pip3] nvidia-cuda-cupti-cu12==12.4.127\n[pip3] nvidia-cuda-nvrtc-cu12==12.4.127\n[pip3] nvidia-cuda-runtime-cu12==12.4.127\n[pip3] nvidia-cudnn-cu12==9.1.0.70\n[pip3] nvidia-cufft-cu12==11.2.1.3\n[pip3] nvidia-curand-cu12==10.3.5.147\n[pip3] nvidia-cusolver-cu12==11.6.1.9\n[pip3] nvidia-cusparse-cu12==12.3.1.170\n[pip3] nvidia-cusparselt-cu12==0.6.2\n[pip3] nvidia-nccl-cu12==2.21.5\n[pip3] nvidia-nvjitlink-cu12==12.4.127\n[pip3] nvidia-nvtx-cu12==12.4.127\n[pip3] torch==2.6.0+cu124\n[pip3] torchaudio==2.6.0+cu124\n[pip3] torchvision==0.21.0+cu124\n[pip3] triton==3.2.0\n[conda] numpy 2.1.2 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.4.5.8 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.1.0.70 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.2.1.3 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.5.147 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.6.1.9 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.3.1.170 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.6.2 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.21.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.4.127 pypi_0 pypi\n[conda] torch 2.6.0+cu124 pypi_0 pypi\n[conda] torchaudio 2.6.0+cu124 pypi_0 pypi\n[conda] torchvision 0.21.0+cu124 pypi_0 pypi\n[conda] triton 3.2.0 pypi_0 pypi", | |
| "transformers_version": "4.48.2", | |
| "upper_git_hash": null, | |
| "tokenizer_pad_token": [ | |
| "<|finetune_right_pad_id|>", | |
| "128004" | |
| ], | |
| "tokenizer_eos_token": [ | |
| "<｜end▁of▁sentence｜>", | |
| "128001" | |
| ], | |
| "tokenizer_bos_token": [ | |
| "<｜begin▁of▁sentence｜>", | |
| "128000" | |
| ], | |
| "eot_token_id": 128001, | |
| "max_length": 131072, | |
| "task_hashes": { | |
| "mmlu_conceptual_physics": "0dab4aa95a4bdc59a8a6e98cb7cf656be66b33532fc207b126e0af3541d6845e", | |
| "mmlu_high_school_chemistry": "07c90c0bdf076d2e1d8c5b5b15cfa314172ba0c038c8f57a1371101ec4c2f032", | |
| "mmlu_college_biology": "63f40ecc7d37a3f6acdcb62249584d145fee87e2bcecc883a99b69329bb7e683", | |
| "mmlu_college_chemistry": "2096ca218774628f8b593299c8d64d29dade2ec0d2d4e8a346d95f9ecc743bba", | |
| "mmlu_machine_learning": "b9b58fe6dc4b136863255cacf87dcf5e93dcbbb6f2ee06e9eaf4109d80efa066", | |
| "mmlu_elementary_mathematics": "f46f9d3a5d18432d83c07dafdc2de44a55eaf465383c2587e562595531d0ceb7", | |
| "mmlu_abstract_algebra": "a9284132783682151e071b6727a93af6e3764e6c8fc1220da083252ec473d9b8", | |
| "mmlu_astronomy": "5182a24822d71d7daeb2c281d0461004a6d884132399ec22f29e1ccd2bfa1c27", | |
| "mmlu_high_school_statistics": "3a6524402d4462efc50bb2fe0f2d2f9d6e5ed12dd8e252b400b6c8d3d3da81fb", | |
| "mmlu_anatomy": "1064a5233bdc4832b0162ddfe506668528fb6f1b9f94ca1664a42d88ec40fa8a", | |
| "mmlu_college_mathematics": "e91cc866f9b065a74630ba3105c1f12407d597278db729c7dca63162bf0550e5", | |
| "mmlu_computer_security": "2b05740014af4b59ca05b275da70a87176923435dde04496cbb2d2aa87ff2b56", | |
| "mmlu_college_computer_science": "6b86afa4833b9b86bbe3fbb5eff3c61e2e7a10a5cb5a0aba9ca26e4c21a5a9b3", | |
| "mmlu_electrical_engineering": "c0c3183e8857378b36ca0e6e3cf0fd7333a88d72f7b65566f9c600d9aaddd407", | |
| "mmlu_college_physics": "339c1587d96ad1c51d61f5786b0c557eb6fd7230a23a1d2f5928a8e82dec8f57", | |
| "mmlu_high_school_computer_science": "41b882dca70082604a9d5a3ca688feb66052244c95ea09118f244d837d025ca7", | |
| "mmlu_high_school_physics": "73b5a0ddf7c6945a7d3781508a60d7d4c47ce5d122dbfdc8a3c5d44678444f29", | |
| "mmlu_high_school_biology": "7923a80586c3edec953f2e17bb9e1bf55c7993c9f2e0cf385d15a23ae227b25e", | |
| "mmlu_high_school_mathematics": "82f26da52a1238bdc468e624633e77789d5712d46bb7cbe13ce3f6035a981ceb", | |
| "mmlu_medical_genetics": "b4d5132944c01b046705e9543160a1c7039f19dc20786e1ad35c119ed95d906c", | |
| "mmlu_global_facts": "f23a2891940181a04cbd03dffe00a019e8f278d0ac713f5b25164dccd10a074b", | |
| "mmlu_marketing": "146202fd0667f90518ea8184a562f470a74b814cfff706282e1540829c5850ec", | |
| "mmlu_college_medicine": "287023df00ce3b6ba701fe7b9516c3526ada8fdab66283db76f500827821d531", | |
| "mmlu_human_aging": "7ecdf334655f2ad6b3bce62962c3b74dd9af6cfef73ec9ed60619850a361049f", | |
| "mmlu_virology": "16831b58c63deadaf215eec84890f8ee921232db13321eabed2798859bc6f0fe", | |
| "mmlu_business_ethics": "f2456d6ac37b427d149bd8377609a5614e6b8294dbb725853da55f3c7ef25dc9", | |
| "mmlu_clinical_knowledge": "91402449cccce57dccfb43b0dc8f1255d7c37c7de2f9b1ef461ddeb3e735f19b", | |
| "mmlu_professional_medicine": "0bf917e544e3dd8900c168649b1c78a0e7da178ddd625875e26e021bec3f363a", | |
| "mmlu_nutrition": "556cde8aabd6e9378a53d0a1d3402e410b376e2904c32fefab384f66c52d5932", | |
| "mmlu_miscellaneous": "d16e1e74dd689977ce82bfbfe2ad46c62007502c1822ad79a92db4eb943e44bf", | |
| "mmlu_professional_accounting": "d453f775093cb1fa13782da7d29e6d528ed2d376775fe341c5a299021d06655e", | |
| "mmlu_management": "4bdaa87fd50c4c833b6535e7f7a843eb9a1db92660c45a7bd68bff0984c00967", | |
| "mmlu_public_relations": "99ce101617b83fdd372586b1c0b2dee2838ba001e9198407c6eb542a075fa975", | |
| "mmlu_sociology": "e95fcaeb9d227b7b165c084ac051117eef2e9096851a60ebd785de35231325be", | |
| "mmlu_security_studies": "56067656c68278fc8e39a319b54becba5dfb05285ef348995989102ac3fc799c", | |
| "mmlu_high_school_government_and_politics": "95756d30b504fb3c60d2d4071fe91ec6fd806a1f137df138e09176d09588927b", | |
| "mmlu_high_school_psychology": "e2971a4156a684f256204bb70f56ad72be8da09f3bee6a8aa6568a60adeddab4", | |
| "mmlu_human_sexuality": "32771cb2655b0eb225dbace51df1529fb3f9224ee9841c34779e9a83b2a8b89c", | |
| "mmlu_us_foreign_policy": "b1eb9ccb442f26423eb949cf492f0efe59176977d92960df9eccf760ca009e73", | |
| "mmlu_high_school_microeconomics": "c421909f7cf3166644056bcc7314fd73fcf468f6ad3c4971715239c9eafc922e", | |
| "mmlu_econometrics": "986b33f7a075a211b1c538484fff34c2581ae76456b8f0320124ee36a08fa7b9", | |
| "mmlu_high_school_macroeconomics": "353e1ed25324ba0c0894692716d1ddd7e481391e019a016be04ae48fdca2e3c2", | |
| "mmlu_high_school_geography": "0ba29c0b02c2c730d5b31c8a5b2742898aa655df3dba495eae204b901b152611", | |
| "mmlu_professional_psychology": "bde5b17209ede6d4c812e93876094af52c38ba8c7e95812746ac4328cf09dbf2", | |
| "mmlu_formal_logic": "a28151a802ccb7b4747eef8ee0ae986a06eb7788553762b4fcb7802e81e96856", | |
| "mmlu_prehistory": "9d48749ae37f69711f5afc1b211b1de3229926fe332599a74ae65a4b963515c2", | |
| "mmlu_world_religions": "62492f6341ed38378bb7809fb14c2566763bad910bb40eb3ecd13fba7d2eef05", | |
| "mmlu_philosophy": "434ffa16ac06e27e0d5e6541eb79c6dcf7b73ae2778d1c23a10882138a9b6470", | |
| "mmlu_high_school_world_history": "b1780672308fbf00e7a512eb2848940c6c23e7068ad6035ab0c200a7f63cd481", | |
| "mmlu_professional_law": "ce43e5be02d7d6827f9e37bd9e6a4e5e2e7763eb54392bf8ef588483e9f31e0a", | |
| "mmlu_high_school_us_history": "f2431b192d6f38181ed2039bb9011fdeab2003cc1e6340f1d979f7343cc54168", | |
| "mmlu_logical_fallacies": "770785c97f03df0d545263a8c4cda55a531caacb94be4639f12253ac613dfb11", | |
| "mmlu_international_law": "f165b2c3d593f4a2e385cb0f882dd2c5280f663ab9b6d0981bd6d170a0178cce", | |
| "mmlu_high_school_european_history": "a00c00d4289b8ac7b11222b59374fa037fc194dbeb6dac29fabbfdccb379e518", | |
| "mmlu_moral_disputes": "3ed549f42e16ef2d38d6a6aaa81ff05ce7e352b5dd186abc526ab42567acc229", | |
| "mmlu_moral_scenarios": "b8d58be4052f8a32d698e0a3a21fef55c89fe5ca61ed3b84de9bdcf2ef61c789", | |
| "mmlu_jurisprudence": "62d87032f8f4a394703371c3586faa41c0c2fee68992b4723e638bd7dbf799aa", | |
| "arc_challenge": "b7d441af53198b01a67fb8bf2269377dfe47da5aa6e569abb31c9d430b6d4fee" | |
| }, | |
| "model_source": "hf", | |
| "model_name": "DeepSeek-R1-Distill-Llama-70B-gptq-4bit", | |
| "model_name_sanitized": "DeepSeek-R1-Distill-Llama-70B-gptq-4bit", | |
| "system_instruction": null, | |
| "system_instruction_sha": null, | |
| "fewshot_as_multiturn": false, | |
| "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜>'}}{% endif %}", | |
| "chat_template_sha": "b6835114b7303ddd78919a82e4d9f7d8c26ed0d7dfc36beeb12d524f6144eab1", | |
| "start_time": 250958.618782607, | |
| "end_time": 257037.417245904, | |
| "total_evaluation_time_seconds": "6078.798463297018" | |
| } |