Add files using upload-large-folder tool

opencompass/configs/datasets/FinanceIQ/FinanceIQ_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .FinanceIQ_gen_e0e6b5 import financeIQ_datasets  # noqa: F401, F403

opencompass/configs/datasets/FinanceIQ/FinanceIQ_gen_e0e6b5.py

@@ -0,0 +1,77 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import FixKRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import FinanceIQDataset
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+financeIQ_subject_mapping_en = {
+    'certified_public_accountant': '注册会计师（CPA）',
+    'banking_qualification': '银行从业资格',
+    'securities_qualification': '证券从业资格',
+    'fund_qualification': '基金从业资格',
+    'insurance_qualification': '保险从业资格CICE',
+    'economic_analyst': '经济师',
+    'taxation_practitioner': '税务师',
+    'futures_qualification': '期货从业资格',
+    'certified_fin_planner': '理财规划师',
+    'actuary_fin_math': '精算师-金融数学',
+}
+
+financeIQ_subject_mapping = {
+    '注册会计师（CPA）': '注册会计师（CPA）',
+    '银行从业资格': '银行从业资格',
+    '证券从业资格': '证券从业资格',
+    '基金从业资格': '基金从业资格',
+    '保险从业资格CICE': '保险从业资格CICE',
+    '经济师': '经济师',
+    '税务师': '税务师',
+    '期货从业资格': '期货从业资格',
+    '理财规划师': '理财规划师',
+    '精算师-金融数学': '精算师-金融数学',
+}
+
+financeIQ_all_sets = list(financeIQ_subject_mapping.keys())
+
+financeIQ_datasets = []
+for _name in financeIQ_all_sets:
+    _ch_name = financeIQ_subject_mapping[_name]
+    financeIQ_infer_cfg = dict(
+        ice_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                begin='</E>',
+                round=[
+                    dict(
+                        role='HUMAN',
+                        prompt=
+                        f'以下是关于{_ch_name}的单项选择题，请直接给出正确答案的选项。\n题目：{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}'
+                    ),
+                    dict(role='BOT', prompt='答案是: {answer}'),
+                ]),
+            ice_token='</E>',
+        ),
+        retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+        inferencer=dict(type=GenInferencer),
+    )
+
+    financeIQ_eval_cfg = dict(
+        evaluator=dict(type=AccEvaluator),
+        pred_postprocessor=dict(type=first_capital_postprocess))
+
+    financeIQ_datasets.append(
+        dict(
+            type=FinanceIQDataset,
+            path='./data/FinanceIQ/',
+            name=_name,
+            abbr=f'FinanceIQ-{_name}',
+            reader_cfg=dict(
+                input_columns=['question', 'A', 'B', 'C', 'D'],
+                output_column='answer',
+                train_split='dev',
+                test_split='test'),
+            infer_cfg=financeIQ_infer_cfg,
+            eval_cfg=financeIQ_eval_cfg,
+        ))
+
+del _name, _ch_name

opencompass/configs/datasets/FinanceIQ/FinanceIQ_ppl.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .FinanceIQ_ppl_42b9bd import financeIQ_datasets  # noqa: F401, F403

opencompass/configs/datasets/FinanceIQ/FinanceIQ_ppl_42b9bd.py

@@ -0,0 +1,76 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import FixKRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import FinanceIQDataset
+
+financeIQ_subject_mapping_en = {
+    'certified_public_accountant': '注册会计师（CPA）',
+    'banking_qualification': '银行从业资格',
+    'securities_qualification': '证券从业资格',
+    'fund_qualification': '基金从业资格',
+    'insurance_qualification': '保险从业资格CICE',
+    'economic_analyst': '经济师',
+    'taxation_practitioner': '税务师',
+    'futures_qualification': '期货从业资格',
+    'certified_fin_planner': '理财规划师',
+    'actuary_fin_math': '精算师-金融数学',
+}
+
+financeIQ_subject_mapping = {
+    '注册会计师（CPA）': '注册会计师（CPA）',
+    '银行从业资格': '银行从业资格',
+    '证券从业资格': '证券从业资格',
+    '基金从业资格': '基金从业资格',
+    '保险从业资格CICE': '保险从业资格CICE',
+    '经济师': '经济师',
+    '税务师': '税务师',
+    '期货从业资格': '期货从业资格',
+    '理财规划师': '理财规划师',
+    '精算师-金融数学': '精算师-金融数学',
+}
+
+financeIQ_all_sets = list(financeIQ_subject_mapping.keys())
+
+financeIQ_datasets = []
+for _name in financeIQ_all_sets:
+    _ch_name = financeIQ_subject_mapping[_name]
+    financeIQ_infer_cfg = dict(
+        ice_template=dict(
+            type=PromptTemplate,
+            template={
+                answer: dict(
+                    begin='</E>',
+                    round=[
+                        dict(
+                            role='HUMAN',
+                            prompt=f'以下是关于{_ch_name}的单项选择题，请直接给出正确答案的选项。\n题目：{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}'
+                        ),
+                        dict(role='BOT', prompt=f'答案是: {answer}'),
+                    ])
+                for answer in ['A', 'B', 'C', 'D']
+            },
+            ice_token='</E>',
+        ),
+        retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+        inferencer=dict(type=PPLInferencer),
+    )
+
+    financeIQ_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+    financeIQ_datasets.append(
+        dict(
+            type=FinanceIQDataset,
+            path='./data/FinanceIQ/',
+            name=_name,
+            abbr=f'FinanceIQ-{_name}',
+            reader_cfg=dict(
+                input_columns=['question', 'A', 'B', 'C', 'D'],
+                output_column='answer',
+                train_split='dev',
+                test_split='test'),
+            infer_cfg=financeIQ_infer_cfg,
+            eval_cfg=financeIQ_eval_cfg,
+        ))
+
+del _name, _ch_name

opencompass/configs/datasets/MathBench/deprecated_mathbench_gen_7b734b.py

@@ -0,0 +1,110 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import CircularEvaluator, AccEvaluator
+from opencompass.datasets import MathBenchDataset, mathbench_postprocess
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+
+single_choice_prompts = {
+    'single_choice_cn_with_reasoning': '以下是一道关于数学的单项选择题，请你一步一步推理，并在最后用“所以答案为选项X”给出答案，其中“X”为选项A，B，C，D中你认为正确的选项。下面是你要回答的问题\n{question}\n让我们一步一步思考：\n',
+    'single_choice_cn': '以下是一道关于数学的单项选择题，请你直接回答正确答案的选项序号。\n下面是你要回答的题目：\n{question}\n答案选项：',
+    'single_choice_en_with_reasoning': "Here is a multiple-choice question about mathematics. Please reason through it step by step, and at the end, provide your answer option with 'Therefore, the correct answer is option X', Where 'X' is the correct option you think from A，B，C，D. Here is the question you need to answer:\n{question}\nLet's think step by step:",
+    'single_choice_en': 'Here is a multiple-choice question about mathematics. Please provide the correct answer option directly.\nHere is the question you need to answer:\n{question}\nAnswer option:',
+}
+
+cloze_prompts = {
+    'cloze_cn': [
+                dict(role='HUMAN', prompt='Q: 林中有15棵树。林务工人员今天将在林中种植树木。完成后，将有21棵树。林务工人员今天种植了多少棵树？'),
+                dict(role='BOT', prompt='A: 我们从15棵树开始。后来有21棵树。差值必定是他们种植的树木数量。所以，他们必须种植了21 - 15 = 6棵树。答案是 6\n'),
+                dict(role='HUMAN', prompt='Q: 如果停车场有3辆车，又有2辆车进来，停车场里有多少辆车？'),
+                dict(role='BOT', prompt='A: 停车场已经有3辆车。又进来了2辆车。现在有3 + 2 = 5辆车。答案是 5\n'),
+                dict(role='HUMAN', prompt='Q: 黎恩有32块巧克力，她的妹妹有42块。如果他们吃了35块，他们总共剩下多少块？'),
+                dict(role='BOT', prompt='A: 黎恩有32块巧克力，Leah的妹妹有42块。这意味着原本有32 + 42 = 74块巧克力。被吃掉了35块。所以他们总共还剩下74 - 35 = 39块巧克力。答案是 39\n'),
+                dict(role='HUMAN', prompt='Q: 杰森有20个棒棒糖。他给丹妮一些棒棒糖。现在Jason只剩下12个棒棒糖。杰森给丹妮多少个棒棒糖？'),
+                dict(role='BOT', prompt='A: 杰森有20个棒棒糖。因为他现在只剩下12个，所以他必须把剩下的都给了丹妮。他给丹妮的棒棒糖数量必定是20 - 12 = 8个。答案是 8\n'),
+                dict(role='HUMAN', prompt='Q: 莎莎有五个玩具。在圣诞节，他从他的爸爸和妈妈那里各得到了两个玩具。现在他有多少个玩具？'),
+                dict(role='BOT', prompt='A: 她有5个玩具。他从妈妈那里得到了2个，所以之后他有5 + 2 = 7个玩具。然后他从爸爸那里得到了2个，所以总共他有7 + 2 = 9个玩具。答案是 9\n'),
+                dict(role='HUMAN', prompt='Q: 服务器房间里有九台电脑。从周一到周四每天增加五台电脑。现在服务器房里有多少台电脑？'),
+                dict(role='BOT', prompt='A: 从周一到周四有4天。每天增加5台电脑。这意味着总共增加了4 * 5 = 20台电脑。一开始有9台电脑，所以现在有9 + 20 = 29台电脑。答案是 29\n'),
+                dict(role='HUMAN', prompt='Q: 迈克尔有58个高尔夫球。星期二，他丢失了23个高尔夫球。星期三，他又丢失了2个。星期三结束时他还剩下多少个高尔夫球？'),
+                dict(role='BOT', prompt='A: 迈克尔一开始有58个球。星期二他丢失了23个，所以之后他还剩下58 - 23 = 35个球。星期三他又丢失了2个，所以现在他还剩下35 - 2 = 33个球。答案是 33\n'),
+                dict(role='HUMAN', prompt='Q: 奥利弗有23美元。她用每个3美元的价格买了五个百吉饼。她还剩下多少钱？'),
+                dict(role='BOT', prompt='A: 她以每个3美元的价格买了5个百吉饼。这意味着她在百吉饼上花费了5 * 3 = 15美元。她一开始有23美元，所以现在她还剩下23 - 15 = 8美元。答案是 8\n'),
+                dict(role='HUMAN', prompt='Q: {question}'),
+                dict(role='BOT', prompt='A: {answer}'),
+                ],
+    'cloze_en': [
+                dict(role='HUMAN', prompt='Q: There are 15 trees in the grove. Grove workers will plant trees in the grove today. After they are done, there will be 21 trees. How many trees did the grove workers plant today?'),
+                dict(role='BOT', prompt='A: We start with 15 trees. Later we have 21 trees. The difference must be the number of trees they planted. So, they must have planted 21 - 15 = 6 trees. The answer is 6.\n'),
+                dict(role='HUMAN', prompt='Q: If there are 3 cars in the parking lot and 2 more cars arrive, how many cars are in the parking lot?'),
+                dict(role='BOT', prompt='A: There are 3 cars in the parking lot already. 2 more arrive. Now there are 3 + 2 = 5 cars. The answer is 5.\n'),
+                dict(role='HUMAN', prompt='Q: Leah had 32 chocolates and her sister had 42. If they ate 35, how many pieces do they have left in total?'),
+                dict(role='BOT', prompt="A: Leah had 32 chocolates and Leah's sister had 42. That means there were originally 32 + 42 = 74 chocolates. 35 have been eaten. So in total they still have 74 - 35 = 39 chocolates. The answer is 39.\n"),
+                dict(role='HUMAN', prompt='Q: Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 lollipops. How many lollipops did Jason give to Denny?'),
+                dict(role='BOT', prompt='A: Jason had 20 lollipops. Since he only has 12 now, he must have given the rest to Denny. The number of lollipops he has given to Denny must have been 20 - 12 = 8 lollipops. The answer is 8.\n'),
+                dict(role='HUMAN', prompt='Q: Shawn has five toys. For Christmas, he got two toys each from his mom and dad. How many toys does he have now?'),
+                dict(role='BOT', prompt='A: He has 5 toys. He got 2 from mom, so after that he has 5 + 2 = 7 toys. Then he got 2 more from dad, so in total he has 7 + 2 = 9 toys. The answer is 9.\n'),
+                dict(role='HUMAN', prompt='Q: There were nine computers in the server room. Five more computers were installed each day, from monday to thursday. How many computers are now in the server room?'),
+                dict(role='BOT', prompt='A: There are 4 days from monday to thursday. 5 computers were added each day. That means in total 4 * 5 = 20 computers were added. There were 9 computers in the beginning, so now there are 9 + 20 = 29 computers. The answer is 29.\n'),
+                dict(role='HUMAN', prompt='Q: Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. How many golf balls did he have at the end of wednesday?'),
+                dict(role='BOT', prompt='A: Michael initially had 58 balls. He lost 23 on Tuesday, so after that he has 58 - 23 = 35 balls. On Wednesday he lost 2 more so now he has 35 - 2 = 33 balls. The answer is 33.\n'),
+                dict(role='HUMAN', prompt='Q: Olivia has $23. She bought five bagels for $3 each. How much money does she have left?'),
+                dict(role='BOT', prompt='A: She bought 5 bagels for $3 each. This means she spent 5 * $3 = $15 on the bagels. She had $23 in beginning, so now she has $23 - $15 = $8. The answer is 8.\n'),
+                dict(role='HUMAN', prompt='Q: {question}'),
+                dict(role='BOT', prompt='A: {answer}\n'),
+]}
+cloze_prompts['refine_cloze_cn'] = cloze_prompts['cloze_cn']
+
+mathbench_sets = {
+    'college': ['single_choice_cn', 'cloze_en'],
+    'high': ['single_choice_cn', 'single_choice_en'],
+    'middle': ['single_choice_cn'],
+    'primary': ['cloze_cn'],
+    'primary_refine': ['refine_cloze_cn']
+}
+
+# Generate reasoning path or not, only for single choice
+with_reasoning = False
+
+# Use circular evaluation or not
+with_circular_eval = True
+
+mathbench_datasets = []
+
+for _split in list(mathbench_sets.keys()):
+    for _name in mathbench_sets[_split]:
+        mathbench_infer_cfg = dict(
+            prompt_template=dict(
+                type=PromptTemplate,
+                template=dict(
+                    round=[
+                        dict(
+                            role='HUMAN',
+                            prompt=single_choice_prompts[_name + '_with_reasoning'] if with_reasoning else single_choice_prompts[_name],
+                        ),
+                        dict(role='BOT', prompt='{answer}')] if 'choice' in _name else cloze_prompts[_name],
+                    ),
+            ),
+            retriever=dict(type=ZeroRetriever),
+            inferencer=dict(type=GenInferencer, max_out_len=512),
+        )
+
+        mathbench_eval_cfg = dict(
+            evaluator=dict(type=CircularEvaluator if 'choice' in _name and with_circular_eval else AccEvaluator),
+            pred_postprocessor=dict(type=first_option_postprocess, options='ABCD') if 'single_choice' in _name else dict(type=mathbench_postprocess, name=_name))
+
+        mathbench_datasets.append(
+            dict(
+                abbr='mathbench-' + _split + '-' + _name,
+                type=MathBenchDataset,
+                path=f'./data/mathbench/{_split}',
+                name=_name,
+                with_circular=with_circular_eval,
+                reader_cfg=dict(
+                    input_columns=['question'],
+                    output_column='answer'
+                    ),
+                infer_cfg=mathbench_infer_cfg,
+                eval_cfg=mathbench_eval_cfg,
+            ))

opencompass/configs/datasets/MathBench/mathbench_2024_few_shot_mixed_4a3fd4.py

@@ -0,0 +1,81 @@
+from mmengine.config import read_base
+from copy import deepcopy
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer, PPLInferencer
+from opencompass.openicl.icl_evaluator import CircularEvaluator, AccEvaluator
+from opencompass.datasets import MathBenchDataset, mathbench_postprocess
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+with read_base():
+    from .mathbench_prompt import zero_shot_prompts, few_shot_prompts, mathbench_sets
+
+# Max for this dataset is 4
+num_shot = 4
+# Generate reasoning path or not, only for single choice
+with_reasoning = True
+# Use circular evaluation or not
+with_circular_eval = True
+# Use PPL mode in single choice test or not
+use_ppl_single_choice = True
+
+assert 0 <= num_shot <= 4
+if num_shot == 0:
+    prompts = zero_shot_prompts
+else:
+    prompts = {name: p[- 2 * num_shot - 2:] for name, p in few_shot_prompts.items()}
+
+mathbench_datasets = []
+for _split in mathbench_sets:
+    for _name in mathbench_sets[_split]:
+        if 'single_choice' in _name:
+            if with_reasoning and not use_ppl_single_choice:
+                template_round = prompts[_name + '_with_reasoning']
+            else:
+                template_round = prompts[_name]
+        else:
+            template_round = prompts[_name]
+
+        if 'single_choice' in _name:
+            pred_postprocessor = dict(type=first_option_postprocess, options='ABCD')
+        else:
+            pred_postprocessor = dict(type=mathbench_postprocess, name=_name)
+
+        if 'single_choice' in _name and with_circular_eval:
+            evaluator = dict(type=CircularEvaluator)
+        else:
+            evaluator = dict(type=AccEvaluator)
+
+        # assemble the final config
+        mathbench_reader_cfg = dict(input_columns=['question'], output_column='answer')
+        if use_ppl_single_choice and 'single_choice' in _name:
+            template = {}
+            for answer in ['A', 'B', 'C', 'D']:
+                one_template_round = deepcopy(template_round)
+                one_template_round[-1]['prompt'] = one_template_round[-1]['prompt'].format(answer=answer)
+                template[answer] = dict(round=one_template_round)
+            mathbench_infer_cfg = dict(
+                prompt_template=dict(type=PromptTemplate, template=template),
+                retriever=dict(type=ZeroRetriever),
+                inferencer=dict(type=PPLInferencer),
+            )
+        else:
+            mathbench_infer_cfg = dict(
+                prompt_template=dict(type=PromptTemplate, template=dict(round=template_round)),
+                retriever=dict(type=ZeroRetriever),
+                inferencer=dict(type=GenInferencer, max_out_len=2048, stopping_criteria=['Question:']),
+            )
+        mathbench_eval_cfg = dict(evaluator=evaluator, pred_postprocessor=pred_postprocessor)
+
+        mathbench_datasets.append(
+            dict(
+                abbr='mathbench-' + _split + '-' + _name,
+                type=MathBenchDataset,
+                path=f'data/mathbench_v1/{_split}',
+                name=_name,
+                with_circular=with_circular_eval,
+                reader_cfg=mathbench_reader_cfg,
+                infer_cfg=mathbench_infer_cfg,
+                eval_cfg=mathbench_eval_cfg,
+            )
+        )

opencompass/configs/datasets/MathBench/mathbench_2024_gen_19e486.py

@@ -0,0 +1,114 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import CircularEvaluator, AccEvaluator
+from opencompass.datasets import MathBenchDataset, mathbench_postprocess
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+single_choice_prompts = {
+    'single_choice_cn_with_reasoning': '以下是一道关于数学的单项选择题，请你一步一步推理，并在最后用“所以答案为选项X”给出答案，其中“X”为选项A，B，C，D中你认为正确的选项。下面是你要回答的问题\n{question}\n让我们一步一步思考：\n',
+    'single_choice_cn': '以下是一道关于数学的单项选择题，请你直接回答正确答案的选项序号。\n下面是你要回答的题目：\n{question}\n答案选项：',
+    'single_choice_en_with_reasoning': "Here is a multiple-choice question about mathematics. Please reason through it step by step, and at the end, provide your answer option with 'Therefore, the correct answer is option X', Where 'X' is the correct option you think from A，B，C，D. Here is the question you need to answer:\n{question}\nLet's think step by step:",
+    'single_choice_en': 'Here is a multiple-choice question about mathematics. Please provide the correct answer option directly.\nHere is the question you need to answer:\n{question}\nAnswer option:',
+}
+
+cloze_prompts = {
+    'cloze_cn': [
+                dict(role='HUMAN', prompt='Q: 林中有15棵树。林务工人员今天将在林中种植树木。完成后，将有21棵树。林务工人员今天种植了多少棵树？'),
+                dict(role='BOT', prompt='A: 我们从15棵树开始。后来有21棵树。差值必定是他们种植的树木数量。所以，他们必须种植了21 - 15 = 6棵树。答案是 6\n'),
+                dict(role='HUMAN', prompt='Q: 如果停车场有3辆车，又有2辆车进来，停车场里有多少辆车？'),
+                dict(role='BOT', prompt='A: 停车场已经有3辆车。又进来了2辆车。现在有3 + 2 = 5辆车。答案是 5\n'),
+                dict(role='HUMAN', prompt='Q: 黎恩有32块巧克力，她的妹妹有42块。如果他们吃了35块，他们总共剩下多少块？'),
+                dict(role='BOT', prompt='A: 黎恩有32块巧克力，Leah的妹妹有42块。这意味着原本有32 + 42 = 74块巧克力。被吃掉了35块。所以他们总共还剩下74 - 35 = 39块巧克力。答案是 39\n'),
+                dict(role='HUMAN', prompt='Q: 杰森有20个棒棒糖。他给丹妮一些棒棒糖。现在Jason只剩下12个棒棒糖。杰森给丹妮多少个棒棒糖？'),
+                dict(role='BOT', prompt='A: 杰森有20个棒棒糖。因为他现在只剩下12个，所以他必须把剩下的都给了丹妮。他给丹妮的棒棒糖数量必定是20 - 12 = 8个。答案是 8\n'),
+                dict(role='HUMAN', prompt='Q: 莎莎有五个玩具。在圣诞节，他从他的爸爸和妈妈那里各得到了两个玩具。现在他有多少个玩具？'),
+                dict(role='BOT', prompt='A: 她有5个玩具。他从妈妈那里得到了2个，所以之后他有5 + 2 = 7个玩具。然后他从爸爸那里得到了2个，所以总共他有7 + 2 = 9个玩具。答案是 9\n'),
+                dict(role='HUMAN', prompt='Q: 服务器房间里有九台电脑。从周一到周四每天增加五台电脑。现在服务器房里有多少台电脑？'),
+                dict(role='BOT', prompt='A: 从周一到周四有4天。每天增加5台电脑。这意味着总共增加了4 * 5 = 20台电脑。一开始有9台电脑，所以现在有9 + 20 = 29台电脑。答案是 29\n'),
+                dict(role='HUMAN', prompt='Q: 迈克尔有58个高尔夫球。星期二，他丢失了23个高尔夫球。星期三，他又丢失了2个。星期三结束时他还剩下多少个高尔夫球？'),
+                dict(role='BOT', prompt='A: 迈克尔一开始有58个球。星期二他丢失了23个，所以之后他还剩下58 - 23 = 35个球。星期三他又丢失了2个，所以现在他还剩下35 - 2 = 33个球。答案是 33\n'),
+                dict(role='HUMAN', prompt='Q: 奥利弗有23美元。她用每个3美元的价格买了五个百吉饼。她还剩下多少钱？'),
+                dict(role='BOT', prompt='A: 她以每个3美元的价格买了5个百吉饼。这意味着她在百吉饼上花费了5 * 3 = 15美元。她一开始有23美元，所以现在她还剩下23 - 15 = 8美元。答案是 8\n'),
+                dict(role='HUMAN', prompt='Q: {question}'),
+                dict(role='BOT', prompt='A: {answer}'),
+                ],
+    'cloze_en': [
+                dict(role='HUMAN', prompt='Q: There are 15 trees in the grove. Grove workers will plant trees in the grove today. After they are done, there will be 21 trees. How many trees did the grove workers plant today?'),
+                dict(role='BOT', prompt='A: We start with 15 trees. Later we have 21 trees. The difference must be the number of trees they planted. So, they must have planted 21 - 15 = 6 trees. The answer is 6.\n'),
+                dict(role='HUMAN', prompt='Q: If there are 3 cars in the parking lot and 2 more cars arrive, how many cars are in the parking lot?'),
+                dict(role='BOT', prompt='A: There are 3 cars in the parking lot already. 2 more arrive. Now there are 3 + 2 = 5 cars. The answer is 5.\n'),
+                dict(role='HUMAN', prompt='Q: Leah had 32 chocolates and her sister had 42. If they ate 35, how many pieces do they have left in total?'),
+                dict(role='BOT', prompt="A: Leah had 32 chocolates and Leah's sister had 42. That means there were originally 32 + 42 = 74 chocolates. 35 have been eaten. So in total they still have 74 - 35 = 39 chocolates. The answer is 39.\n"),
+                dict(role='HUMAN', prompt='Q: Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 lollipops. How many lollipops did Jason give to Denny?'),
+                dict(role='BOT', prompt='A: Jason had 20 lollipops. Since he only has 12 now, he must have given the rest to Denny. The number of lollipops he has given to Denny must have been 20 - 12 = 8 lollipops. The answer is 8.\n'),
+                dict(role='HUMAN', prompt='Q: Shawn has five toys. For Christmas, he got two toys each from his mom and dad. How many toys does he have now?'),
+                dict(role='BOT', prompt='A: He has 5 toys. He got 2 from mom, so after that he has 5 + 2 = 7 toys. Then he got 2 more from dad, so in total he has 7 + 2 = 9 toys. The answer is 9.\n'),
+                dict(role='HUMAN', prompt='Q: There were nine computers in the server room. Five more computers were installed each day, from monday to thursday. How many computers are now in the server room?'),
+                dict(role='BOT', prompt='A: There are 4 days from monday to thursday. 5 computers were added each day. That means in total 4 * 5 = 20 computers were added. There were 9 computers in the beginning, so now there are 9 + 20 = 29 computers. The answer is 29.\n'),
+                dict(role='HUMAN', prompt='Q: Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. How many golf balls did he have at the end of wednesday?'),
+                dict(role='BOT', prompt='A: Michael initially had 58 balls. He lost 23 on Tuesday, so after that he has 58 - 23 = 35 balls. On Wednesday he lost 2 more so now he has 35 - 2 = 33 balls. The answer is 33.\n'),
+                dict(role='HUMAN', prompt='Q: Olivia has $23. She bought five bagels for $3 each. How much money does she have left?'),
+                dict(role='BOT', prompt='A: She bought 5 bagels for $3 each. This means she spent 5 * $3 = $15 on the bagels. She had $23 in beginning, so now she has $23 - $15 = $8. The answer is 8.\n'),
+                dict(role='HUMAN', prompt='Q: {question}'),
+                dict(role='BOT', prompt='A: {answer}\n'),
+]}
+
+mathbench_sets = {
+    # Practice Part
+    'college': ['single_choice_cn', 'single_choice_en'],
+    'high': ['single_choice_cn', 'single_choice_en'],
+    'middle': ['single_choice_cn', 'single_choice_en'],
+    'primary': ['cloze_cn', 'cloze_en'],
+    'arithmetic': ['cloze_en'],
+    # Theory part
+    'college_knowledge': ['single_choice_cn','single_choice_en'],
+    'high_knowledge': ['single_choice_cn','single_choice_en'],
+    'middle_knowledge': ['single_choice_cn','single_choice_en'],
+    'primary_knowledge': ['single_choice_cn','single_choice_en'],
+}
+
+# Generate reasoning path or not, only for single choice
+with_reasoning = True
+
+# Use circular evaluation or not
+with_circular_eval = True
+
+mathbench_datasets = []
+
+for _split in list(mathbench_sets.keys()):
+    for _name in mathbench_sets[_split]:
+        mathbench_infer_cfg = dict(
+            prompt_template=dict(
+                type=PromptTemplate,
+                template=dict(
+                    round=[
+                        dict(
+                            role='HUMAN',
+                            prompt=single_choice_prompts[_name + '_with_reasoning'] if with_reasoning else single_choice_prompts[_name],
+                        ),
+                        dict(role='BOT', prompt='{answer}')] if 'choice' in _name else cloze_prompts[_name],
+                    ),
+            ),
+            retriever=dict(type=ZeroRetriever),
+            inferencer=dict(type=GenInferencer, max_out_len=2048),
+        )
+
+        mathbench_eval_cfg = dict(
+            evaluator=dict(type=CircularEvaluator if 'choice' in _name and with_circular_eval else AccEvaluator),
+            pred_postprocessor=dict(type=first_option_postprocess, options='ABCD') if 'single_choice' in _name else dict(type=mathbench_postprocess, name=_name))
+
+        mathbench_datasets.append(
+            dict(
+                abbr='mathbench-' + _split + '-' + _name,
+                type=MathBenchDataset,
+                path=f'./data/mathbench_v1/{_split}',
+                name=_name,
+                with_circular=with_circular_eval,
+                reader_cfg=dict(
+                    input_columns=['question'],
+                    output_column='answer'
+                    ),
+                infer_cfg=mathbench_infer_cfg,
+                eval_cfg=mathbench_eval_cfg,
+            ))

opencompass/configs/datasets/MathBench/mathbench_2024_gen_1dc21d.py

@@ -0,0 +1,81 @@
+from mmengine.config import read_base
+from copy import deepcopy
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer, PPLInferencer
+from opencompass.openicl.icl_evaluator import CircularEvaluator, AccEvaluator
+from opencompass.datasets import MathBenchDataset, mathbench_postprocess
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+with read_base():
+    from .mathbench_prompt import zero_shot_prompts, few_shot_prompts, mathbench_sets
+
+# Max for this dataset is 4
+num_shot = 4
+# Generate reasoning path or not, only for single choice
+with_reasoning = False
+# Use circular evaluation or not
+with_circular_eval = True
+# Use PPL mode in single choice test or not
+use_ppl_single_choice = False
+
+assert 0 <= num_shot <= 4
+if num_shot == 0:
+    prompts = zero_shot_prompts
+else:
+    prompts = {name: p[- 2 * num_shot - 2:] for name, p in few_shot_prompts.items()}
+
+mathbench_datasets = []
+for _split in mathbench_sets:
+    for _name in mathbench_sets[_split]:
+        if 'single_choice' in _name:
+            if with_reasoning:
+                template_round = prompts[_name + '_with_reasoning']
+            else:
+                template_round = prompts[_name]
+        else:
+            template_round = prompts[_name]
+
+        if 'single_choice' in _name:
+            pred_postprocessor = dict(type=first_option_postprocess, options='ABCD')
+        else:
+            pred_postprocessor = dict(type=mathbench_postprocess, name=_name)
+
+        if 'single_choice' in _name and with_circular_eval:
+            evaluator = dict(type=CircularEvaluator)
+        else:
+            evaluator = dict(type=AccEvaluator)
+
+        # assemble the final config
+        mathbench_reader_cfg = dict(input_columns=['question'], output_column='answer')
+        if use_ppl_single_choice and 'single_choice' in _name and not with_reasoning:
+            template = {}
+            for answer in ['A', 'B', 'C', 'D']:
+                one_template_round = deepcopy(template_round)
+                one_template_round['round'][-1]['prompt'] = one_template_round['round'][-1]['prompt'].format(answer=answer)
+                template[answer] = dict(round=one_template_round)
+            mathbench_infer_cfg = dict(
+                prompt_template=dict(type=PromptTemplate, template=template),
+                retriever=dict(type=ZeroRetriever),
+                inferencer=dict(type=PPLInferencer),
+            )
+        else:
+            mathbench_infer_cfg = dict(
+                prompt_template=dict(type=PromptTemplate, template=dict(round=template_round)),
+                retriever=dict(type=ZeroRetriever),
+                inferencer=dict(type=GenInferencer, max_out_len=2048),
+            )
+        mathbench_eval_cfg = dict(evaluator=evaluator, pred_postprocessor=pred_postprocessor)
+
+        mathbench_datasets.append(
+            dict(
+                abbr='mathbench-no_cot-' + _split + '-' + _name,
+                type=MathBenchDataset,
+                path=f'data/mathbench_v1/{_split}',
+                name=_name,
+                with_circular=with_circular_eval,
+                reader_cfg=mathbench_reader_cfg,
+                infer_cfg=mathbench_infer_cfg,
+                eval_cfg=mathbench_eval_cfg,
+            )
+        )

opencompass/configs/datasets/MathBench/mathbench_2024_gen_50a320.py

@@ -0,0 +1,81 @@
+from mmengine.config import read_base
+from copy import deepcopy
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer, PPLInferencer
+from opencompass.openicl.icl_evaluator import CircularEvaluator, AccEvaluator
+from opencompass.datasets import MathBenchDataset, math_postprocess_v2
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+with read_base():
+    from .mathbench_prompt import zero_shot_prompts, few_shot_prompts, mathbench_sets
+
+# Max for this dataset is 4
+num_shot = 0
+# Generate reasoning path or not, only for single choice
+with_reasoning = True
+# Use circular evaluation or not
+with_circular_eval = True
+# Use PPL mode in single choice test or not
+use_ppl_single_choice = False
+
+assert 0 <= num_shot <= 4
+if num_shot == 0:
+    prompts = zero_shot_prompts
+else:
+    prompts = {name: p[- 2 * num_shot - 2:] for name, p in few_shot_prompts.items()}
+
+mathbench_datasets = []
+for _split in mathbench_sets:
+    for _name in mathbench_sets[_split]:
+        if 'single_choice' in _name:
+            if with_reasoning:
+                template_round = prompts[_name + '_with_reasoning']
+            else:
+                template_round = prompts[_name]
+        else:
+            template_round = prompts[_name]
+
+        if 'single_choice' in _name:
+            pred_postprocessor = dict(type=first_option_postprocess, options='ABCD')
+        else:
+            pred_postprocessor = dict(type=math_postprocess_v2)
+
+        if 'single_choice' in _name and with_circular_eval:
+            evaluator = dict(type=CircularEvaluator)
+        else:
+            evaluator = dict(type=AccEvaluator)
+
+        # assemble the final config
+        mathbench_reader_cfg = dict(input_columns=['question'], output_column='answer')
+        if use_ppl_single_choice and 'single_choice' in _name and not with_reasoning:
+            template = {}
+            for answer in ['A', 'B', 'C', 'D']:
+                one_template_round = deepcopy(template_round)
+                one_template_round['round'][-1]['prompt'] = one_template_round['round'][-1]['prompt'].format(answer=answer)
+                template[answer] = dict(round=one_template_round)
+            mathbench_infer_cfg = dict(
+                prompt_template=dict(type=PromptTemplate, template=template),
+                retriever=dict(type=ZeroRetriever),
+                inferencer=dict(type=PPLInferencer),
+            )
+        else:
+            mathbench_infer_cfg = dict(
+                prompt_template=dict(type=PromptTemplate, template=dict(round=template_round)),
+                retriever=dict(type=ZeroRetriever),
+                inferencer=dict(type=GenInferencer, max_out_len=2048),
+            )
+        mathbench_eval_cfg = dict(evaluator=evaluator, pred_postprocessor=pred_postprocessor)
+
+        mathbench_datasets.append(
+            dict(
+                abbr='mathbench-' + _split + '-' + _name,
+                type=MathBenchDataset,
+                path=f'data/mathbench_v1/{_split}',
+                name=_name,
+                with_circular=with_circular_eval,
+                reader_cfg=mathbench_reader_cfg,
+                infer_cfg=mathbench_infer_cfg,
+                eval_cfg=mathbench_eval_cfg,
+            )
+        )

opencompass/configs/datasets/MathBench/mathbench_prompt.py

@@ -0,0 +1,109 @@
+zero_shot_prompts = {
+    'single_choice_cn_with_reasoning': [
+        dict(role='HUMAN', prompt='问题: 以下是一道关于数学的单项选择题，请你一步一步推理，并在最后用“所以答案为选项X”给出答案，其中“X”为选项A，B，C，D中你认为正确的选项。下面是你要回答的问题\n{question}\n让我们一步一步思考：'),
+    ],
+    'single_choice_cn': [
+        dict(role='HUMAN', prompt='问题: 以下是一道关于数学的单项选择题，请你直接回答正确答案的选项序号。\n下面是你要回答的题目：\n{question}\n所以答案是：'),
+    ],
+    'single_choice_en_with_reasoning': [
+        dict(role='HUMAN', prompt='Question: Here is a multiple-choice question about mathematics. Please reason through it step by step, and at the end, provide your answer option with "Therefore, the correct answer is option X", Where "X" is the correct option you think from A，B，C，D. Here is the question you need to answer:\n{question}\nLet\'s think step by step:'),
+    ],
+    'single_choice_en': [
+        dict(role='HUMAN', prompt='Question: Here is a multiple-choice question about mathematics. Please provide the correct answer option directly.\nHere is the question you need to answer:\n{question}\nThe answer is:'),
+    ],
+    'cloze_en': [
+        dict(role='HUMAN', prompt='{question}\nPlease reason step by step, and put your final answer within \\boxed{}.'),
+    ],
+    'cloze_cn': [
+        dict(role='HUMAN', prompt='{question}\n请一步一步推理，并在最后用\\boxed{}给出你的答案。'),
+    ]
+}
+
+few_shot_prompts = {
+    'single_choice_cn': [
+        dict(role='HUMAN', prompt='问题: 已知i是虚数单位，z为复数，$2+\\frac{1}{i}=z(3+i)$，则在复平面内z对应的点位于____?\nA. 第一象限\nB. 第二象限\nC. 第三象限\nD. 第四象限'),
+        dict(role='BOT', prompt='回答: D'),
+        dict(role='HUMAN', prompt='问题: 将函数$y=\\tan(\\omega x-1)(\\omega>0)$的图像向左平移2个单位长度后,与函数$y=\\tan(\\omega x+3)$的图象重合,则的最小值等于____\nA. $2-\\frac{\\pi}{2}$\nB. 1\nC. $\\pi - 2$\nD. 2'),
+        dict(role='BOT', prompt='回答: D'),
+        dict(role='HUMAN', prompt='问题: 设$（1+2i）a+b=2i$，其中a,b为实数，则（  ）\nA. $a=1,b=-1$\nB. $a=1,b=1$\nC. $a=-1,b=1$\nD. $a=-1,b=-1$'),
+        dict(role='BOT', prompt='回答: A'),
+        dict(role='HUMAN', prompt='问题: 圆锥的底面半径为2，高为4.一个圆柱的下底面在圆锥的底面上，上底面的圆周在圆锥的侧面上，当圆柱侧面积为$4 \\pi$时，该圆柱的体积为____\nA. $\\pi$\nB. $2\\pi$\nC. $3\\pi$\nD. $4\\pi$'),
+        dict(role='BOT', prompt='回答: B'),
+        dict(role='HUMAN', prompt='问题: {question}'),
+        dict(role='BOT', prompt='回答: {answer}'),
+    ],
+    'single_choice_cn_with_reasoning' : [
+        dict(role='HUMAN', prompt='问题: 已知i是虚数单位，z为复数，$2+\\frac{1}{i}=z(3+i)$，则在复平面内z对应的点位于____\nA. 第一象限\nB. 第二象限\nC. 第三象限\nD. 第四象限'),
+        dict(role='BOT', prompt='回答: 因为首先，我们将等式两边同时乘以$(3 + i)$的共轭复数$(3 - i)$，以便消去分母中的虚数部分：$z = \\frac{2 - i}{3 + i} \\cdot \\frac{3 - i}{3 - i}$，这样做的目的是利用复数乘法的性质，从而消去分母中的虚数部分。我们进行乘法运算：$z = \\frac{(2 - i)(3 - i)}{(3 + i)(3 - i)}$，$z = \\frac{6 - 2i - 3i + i^2}{9 - 3i + 3i - i^2}$，由于$i^2 = -1$，我们可以进一步简化：$z = \\frac{6 - 5i - 1}{9 + 1}$ $z = \\frac{{1}{2} - \\frac{1}{2}i$，在复平面上，这个点位于第四象限，因为它的实部是正的（$\\frac{1}{2}$），而虚部是负的（$-\\frac{1}{2}$）\n答案选项：D'),
+        dict(role='HUMAN', prompt='问题: 将函数$y=\\tan(\\omega x-1)(\\omega>0)$的图像向左平移2个单位长度后,与函数$y=\\tan(\\omega x+3)$的图象重合,则的最小值等于____\nA. $2-\\frac{\\pi}{2}$\nB. 1\nC. $\\pi - 2$\nD. 2'),
+        dict(role='BOT', prompt='回答: 将函数$y=\\tan(\\omega x-1)$（其中$\\omega>0$）的图像向左平移2个单位长度后，得到的新函数为$y=\\tan(\\omega(x+2)-1)$。为了使这个新函数的图像与$y=\\tan(\\omega x+3)$的图像重合，我们需要满足以下条件：\n$\\tan(\\omega(x+2)-1) = \\tan(\\omega x+3)$\n由于正切函数的周期性，我们知道$\\tan(\\theta) = \\tan(\\theta + k\\pi)$，其中$k$是任意整数。因此，我们可以写出：\n$\\omega(x+2)-1 = \\omega x+3 + k\\pi$\n解这个方程，我们得到：$\\omega x + 2\\omega - 1 = \\omega x + 3 + k\\pi$\n$2\\omega - 1 = 3 + k\\pi$\n$\\omega = 2 + \\frac{k\\pi}{2}$\n由于$\\omega>0$，我们需要找到满足条件的最小正整数$k$。当$k=0$时，我们得到$\\omega=2$，这是满足条件的最小正值。因此，最小值等于2。答案选项：D'),
+        dict(role='HUMAN', prompt='问题: 设$（1+2i）a+b=2i$，其中a,b为实数，则（  ）\nA. $a=1,b=-1$\nB. $a=1,b=1$\nC. $a=-1,b=1$\nD. $a=-1,b=-1$'),
+        dict(role='BOT', prompt='回答: 我们可以将这个方程写成：$a + 2ai + b = 2i$，然后将实部和虚部分开：\n$a + b = 0$\n$2a = 2$\n由于$a + b = 0$，我们可以得到$a = -b$。将这个值代入$2a = 2$，我们得到$2(-b) = 2$，因此$b = -1$。因此，$a = 1$。答案选项：A'),
+        dict(role='HUMAN', prompt='问题: 一直角三角形的两边长分别为3和4．则第三边的长为____\nA. 5\nB. $\\sqrt 7$\nC. $\\sqrt 5$\nD. 5或者$\\sqrt 7$'),
+        dict(role='BOT', prompt='回答: 我们可以使用勾股定理来解决这个问题。勾股定理告诉我们，如果一个三角形的两条边长分别为$a$和$b$，那么第三条边的长度为$c = \\sqrt{a^2 + b^2}$。在这个问题中，$a = 3$，$b = 4$，因此$c = \\sqrt{3^2 + 4^2} = \\sqrt{9 + 16} = \\sqrt{25} = 5$。答案选项：A'),
+        dict(role='HUMAN', prompt='问题: {question}'),
+        dict(role='BOT', prompt='回答: {answer}'),
+    ],
+    'single_choice_en': [
+        dict(role='HUMAN', prompt='Question: Given that $i$ is the imaginary unit and $z$ is a complex number, $2 + \\frac{1}{i} = z(3 + i)$, the point corresponding to $z$ in the complex plane is located in the ______\nA. First quadrant\nB. Second quadrant\nC. Third quadrant\nD. Fourth quadrant'),
+        dict(role='BOT', prompt='Response: D'),
+        dict(role='HUMAN', prompt='Question: The graph of the function $y = \\tan(\\omega x - 1)$ is shifted 2 units to the left and coincides with the graph of the function $y = \\tan(\\omega x + 3)$. The minimum value of $\\omega$ is ______\nA. $2 - \\frac{\\pi}{2}$\nB. 1\nC. $\\pi - 2$\nD. 2'),
+        dict(role='BOT', prompt='Response: D'),
+        dict(role='HUMAN', prompt='Question: If $(1 + 2i)a + b = 2i$, where $a$ and $b$ are real numbers, then ______\nA. $a = 1, b = -1$\nB. $a = 1, b = 1$\nC. $a = -1, b = 1$\nD. $a = -1, b = -1$'),
+        dict(role='BOT', prompt='Response: A'),
+        dict(role='HUMAN', prompt='Question: The radius of a cone is 2 and its height is 4. A cylinder has its lower base on the base of the cone and its upper base on the lateral surface of the cone. When the lateral surface area of the cylinder is $4\\pi$, the volume of the cylinder is ______\nA. $\\pi$\nB. $2\\pi$\nC. $3\\pi$\nD. $4\\pi$'),
+        dict(role='BOT', prompt='Response: B'),
+        dict(role='HUMAN', prompt='Question: {question}'),
+        dict(role='BOT', prompt='Response: {answer}'),
+    ],
+    'single_choice_en_with_reasoning': [
+        dict(role='HUMAN', prompt='Question: Given that $i$ is the imaginary unit and $z$ is a complex number, $2 + \\frac{1}{i} = z(3 + i)$, the point corresponding to $z$ in the complex plane is located in the ______\nA. First quadrant\nB. Second quadrant\nC. Third quadrant\nD. Fourth quadrant'),
+        dict(role='BOT', prompt='Response: First, we multiply both sides of the equation by the conjugate of $(3 + i)$: $z = \\frac{2 - i}{3 + i} \\cdot \\frac{3 - i}{3 - i}$. We perform the multiplication: $z = \\frac{(2 - i)(3 - i)}{(3 + i)(3 - i)}$, $z = \\frac{6 - 2i - 3i + i^2}{9 - 3i + 3i - i^2}$. Since $i^2 = -1$, we can simplify this further: $z = \\frac{6 - 5i - 1}{9 + 1}$ $z = \\frac{1}{2} - \\frac{1}{2}i$. In the complex plane, this point is located in the fourth quadrant, because its real part is positive ($\\frac{1}{2}$) and its imaginary part is negative ($-\\frac{1}{2}$)\nAnswer option: D'),
+        dict(role='HUMAN', prompt='Question: The graph of the function $y = \\tan(\\omega x - 1)$ is shifted 2 units to the left and coincides with the graph of the function $y = \\tan(\\omega x + 3)$. The minimum value of $\\omega$ is ______\nA. $2 - \\frac{\\pi}{2}$\nB. 1\nC. $\\pi - 2$\nD. 2'),
+        dict(role='BOT', prompt='Response: In order for the graph of this new function to coincide with the graph of $y = \\tan(\\omega x + 3)$, we need to satisfy the following condition: $\\tan(\\omega(x + 2) - 1) = \\tan(\\omega x + 3)$. Therefore, we can write: $\\omega(x + 2) - 1 = \\omega x + 3 + k\\pi$. Solving this equation, we get: $\\omega x + 2\\omega - 1 = \\omega x + 3 + k\\pi$. $2\\omega - 1 = 3 + k\\pi$. $\\omega = 2 + \\frac{k\\pi}{2}$. Since $\\omega > 0$, we need to find the smallest positive integer $k$ that satisfies the condition. When $k = 0$, we get $\\omega = 2$, which is the smallest positive value that satisfies the condition. Therefore, the minimum value is 2. Answer option: D'),
+        dict(role='HUMAN', prompt='Question: If $(1 + 2i)a + b = 2i$, where $a$ and $b$ are real numbers, then ______\nA. $a = 1, b = -1$\nB. $a = 1, b = 1$\nC. $a = -1, b = 1$\nD. $a = -1, b = -1$'),
+        dict(role='BOT', prompt='Response: We can write this equation as: $a + 2ai + b = 2i$, and then separate the real and imaginary parts: $a + b = 0$. $2a = 2$. Since $a + b = 0$, we can get $a = -b$. Substituting this value into $2a = 2$, we get $2(-b) = 2$, so $b = -1$. Therefore, $a = 1$. Answer option: A'),
+        dict(role='HUMAN', prompt='Question: The radius of a cone is 2 and its height is 4. A cylinder has its lower base on the base of the cone and its upper base on the lateral surface of the cone. When the lateral surface area of the cylinder is $4\\pi$, the volume of the cylinder is ______\nA. $\\pi$\nB. $2\\pi$\nC. $3\\pi$\nD. $4\\pi$'),
+        dict(role='BOT', prompt='Response: We can use the Pythagorean theorem to solve this problem. The Pythagorean theorem tells us that if the two sides of a triangle are $a$ and $b$, then the length of the third side is $c = \\sqrt{a^2 + b^2}$. In this problem, $a = 3$ and $b = 4$, so $c = \\sqrt{3^2 + 4^2} = \\sqrt{9 + 16} = \\sqrt{25} = 5$. Answer option: A'),
+        dict(role='HUMAN', prompt='Question: {question}'),
+        dict(role='BOT', prompt='Response: {answer}'),
+    ],
+    'cloze_cn': [
+        dict(role='HUMAN', prompt='问题: 莎莎有五个玩具。在圣诞节，他从他的爸爸和妈妈那里各得到了两个玩具。现在他有多少个玩具？'),
+        dict(role='BOT', prompt='回答: 她有5个玩具。他从妈妈那里得到了2个，所以之后他有5 + 2 = 7个玩具。然后他从爸爸那里得到了2个，所以总共他有7 + 2 = 9个玩具。答案是 9'),
+        dict(role='HUMAN', prompt='问题: 服务器房间里有九台电脑。从周一到周四每天增加五台电脑。现在服务器房里有多少台电脑？'),
+        dict(role='BOT', prompt='回答: 从周一到周四有4天。每天增加5台电脑。这意味着总共增加了4 * 5 = 20台电脑。一开始有9台电脑，所以现在有9 + 20 = 29台电脑。答案是 29'),
+        dict(role='HUMAN', prompt='问题: 迈克尔有58个高尔夫球。星期二，他丢失了23个高尔夫球。星期三，他又丢失了2个。星期三结束时他还剩下多少个高尔夫球？'),
+        dict(role='BOT', prompt='回答: 迈克尔一开始有58个球。星期二他丢失了23个，所以之后他还剩下58 - 23 = 35个球。星期三他又丢失了2个，所以现在他还剩下35 - 2 = 33个球。答案是 33'),
+        dict(role='HUMAN', prompt='问题: 奥利弗有23美元。她用每个3美元的价格买了五个百吉饼。她还剩下多少钱？'),
+        dict(role='BOT', prompt='回答: 她以每个3美元的价格买了5个百吉饼。这意味着她在百吉饼上花费了5 * 3 = 15美元。她一开始有23美元，所以现在她还剩下23 - 15 = 8美元。答案是 8'),
+        dict(role='HUMAN', prompt='问题: {question}'),
+        dict(role='BOT', prompt='回答: {answer}'),
+    ],
+    'cloze_en': [
+        dict(role='HUMAN', prompt='Question: Shawn has five toys. For Christmas, he got two toys each from his mom and dad. How many toys does he have now?'),
+        dict(role='BOT', prompt='Response: He has 5 toys. He got 2 from mom, so after that he has 5 + 2 = 7 toys. Then he got 2 more from dad, so in total he has 7 + 2 = 9 toys. The answer is 9.'),
+        dict(role='HUMAN', prompt='Question: There were nine computers in the server room. Five more computers were installed each day, from monday to thursday. How many computers are now in the server room?'),
+        dict(role='BOT', prompt='Response: There are 4 days from monday to thursday. 5 computers were added each day. That means in total 4 * 5 = 20 computers were added. There were 9 computers in the beginning, so now there are 9 + 20 = 29 computers. The answer is 29.'),
+        dict(role='HUMAN', prompt='Question: Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. How many golf balls did he have at the end of wednesday?'),
+        dict(role='BOT', prompt='Response: Michael initially had 58 balls. He lost 23 on Tuesday, so after that he has 58 - 23 = 35 balls. On Wednesday he lost 2 more so now he has 35 - 2 = 33 balls. The answer is 33.'),
+        dict(role='HUMAN', prompt='Question: Olivia has $23. She bought five bagels for $3 each. How much money does she have left?'),
+        dict(role='BOT', prompt='Response: She bought 5 bagels for $3 each. This means she spent 5 * $3 = $15 on the bagels. She had $23 in beginning, so now she has $23 - $15 = $8. The answer is 8.'),
+        dict(role='HUMAN', prompt='Question: {question}'),
+        dict(role='BOT', prompt='Response: {answer}'),
+    ],
+}
+
+mathbench_sets = {
+    # Practice Part
+    'college': ['single_choice_cn', 'single_choice_en'],
+    'high': ['single_choice_cn', 'single_choice_en'],
+    'middle': ['single_choice_cn', 'single_choice_en'],
+    'primary': ['cloze_cn', 'cloze_en'],
+    'arithmetic': ['cloze_en'],
+    # Theory part
+    'college_knowledge': ['single_choice_cn', 'single_choice_en'],
+    'high_knowledge': ['single_choice_cn', 'single_choice_en'],
+    'middle_knowledge': ['single_choice_cn', 'single_choice_en'],
+    'primary_knowledge': ['single_choice_cn', 'single_choice_en'],
+}

opencompass/configs/datasets/NPHardEval/NPHardEval_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .NPHardEval_gen_22aac5 import NPHardEval_datasets  # noqa: F401, F403

opencompass/configs/datasets/NPHardEval/NPHardEval_gen_22aac5.py

@@ -0,0 +1,59 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets.NPHardEval import (
+    HardGCPDataset, HardGCPEvaluator,
+    Hard_TSP_Dataset, Hard_TSP_Evaluator,
+    Hard_MSP_Dataset, Hard_MSP_Evaluator,
+    CMP_GCP_D_Dataset, CMP_GCP_D_Evaluator,
+    CMP_TSP_D_Dataset, CMP_TSP_D_Evaluator,
+    CMP_KSP_Dataset, CMP_KSP_Evaluator,
+    P_BSP_Dataset, P_BSP_Evaluator,
+    P_EDP_Dataset, P_EDP_Evaluator,
+    P_SPP_Dataset, P_SPP_Evaluator,
+)
+
+NPHardEval_tasks = [
+    ['hard_GCP', 'GCP', HardGCPDataset, HardGCPEvaluator],
+    ['hard_TSP', 'TSP', Hard_TSP_Dataset, Hard_TSP_Evaluator],
+    ['hard_MSP', 'MSP', Hard_MSP_Dataset, Hard_MSP_Evaluator],
+    ['cmp_GCP_D', 'GCP_Decision', CMP_GCP_D_Dataset, CMP_GCP_D_Evaluator],
+    ['cmp_TSP_D', 'TSP_Decision', CMP_TSP_D_Dataset, CMP_TSP_D_Evaluator],
+    ['cmp_KSP', 'KSP', CMP_KSP_Dataset, CMP_KSP_Evaluator],
+    ['p_BSP', 'BSP', P_BSP_Dataset, P_BSP_Evaluator],
+    ['p_EDP', 'EDP', P_EDP_Dataset, P_EDP_Evaluator],
+    ['p_SPP', 'SPP', P_SPP_Dataset, P_SPP_Evaluator],
+]
+
+NPHardEval_datasets = []
+for name, path_name, dataset, evaluator in NPHardEval_tasks:
+    NPHardEval_reader_cfg = dict(input_columns=['prompt', 'level'], output_column='q')
+
+    NPHardEval_infer_cfg = dict(
+        ice_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                begin='</E>',
+                round=[
+                    dict(role='HUMAN', prompt='</E>{prompt}'),
+                    dict(role='BOT', prompt=''),
+                ],
+            ),
+            ice_token='</E>',
+        ),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer),
+    )
+
+    NPHardEval_eval_cfg = dict(evaluator=dict(type=evaluator), pred_role='BOT')
+
+    NPHardEval_datasets.append(
+        dict(
+            type=dataset,
+            abbr=name,
+            path=f'./data/NPHardEval/{path_name}/',
+            reader_cfg=NPHardEval_reader_cfg,
+            infer_cfg=NPHardEval_infer_cfg,
+            eval_cfg=NPHardEval_eval_cfg,
+        )
+    )

opencompass/configs/datasets/NPHardEval/README.md

@@ -0,0 +1,126 @@
+# NPHardEval
+## Introduction
+The following introduction comes from the abstract in [NPHardEval: Dynamic Benchmark on Reasoning Ability of Large Language Models via Complexity Classes](https://arxiv.org/abs/2312.14890)
+
+```
+This benchmark is designed to evaluate the reasoning abilities of LLMs across a broad spectrum of 900 algorithmic questions, extending up to the NP-Hard complexity class. These questions are meticulously chosen to represent a wide range of complexity class below the NP-hard complexity class, offering a rigorous measure of the reasoning ability of LLMs.
+```
+
+## Official link
+
+### Paper
+
+[NPHardEval: Dynamic Benchmark on Reasoning Ability of Large Language Models via Complexity Classes](https://arxiv.org/abs/2312.14890)
+
+### Repository
+
+[NPHardEval](https://github.com/casmlab/NPHardEval)
+
+## Examples
+Input example I:
+```
+The Graph Coloring Problem (Decision Version, GCP-D) involves determining if it is possible to color the vertices of a graph using a given number of colors, ensuring no two adjacent vertices have the same color.
+Find out if the vertices of a graph with 6 vertices can be colored using only 3 colors, such that no adjacent vertices share the same color.
+Provide a yes or no answer, along with a concise explanation of your reasoning. Keep your explanation focused and brief.
+Enclose your output in <root></root> tags. Include your reasoning in <reasoning></reasoning> tags and the final yes/no answer in <final_answer></final_answer> tags, like <final_answer>{'Feasible': 'YES_OR_NO'}</final_answer>.
+
+ The graph is below:
+Vertex 2 is connected to vertex 4.
+Vertex 2 is connected to vertex 5.
+Vertex 3 is connected to vertex 5.
+Vertex 4 is connected to vertex 5.
+Vertex 5 is connected to vertex 6.
+```
+
+Output example I (from GPT-4):
+```
+<root>
+<final_answer>{'Feasible': 'YES'}</final_answer>
+<reasoning>Yes, it is possible to color the vertices of the given graph with 6 vertices using only 3 colors, such that no adjacent vertices share the same color. The graph does not contain any triangles or cycles of length greater than 3, which means it is a planar graph. By the Four Color Theorem, any planar graph can be colored using at most 4 colors. Since we have only 3 colors available, it is feasible to color the vertices without any adjacent vertices sharing the same color.</reasoning>
+</root>
+```
+
+Input example II:
+```
+The Traveling Salesman Problem (Decision Version, TSP-D) focuses on determining if a salesman can complete a route, visiting each city at least once, with the total travel distance being less than a specified value.
+Check if it's possible for a salesman to visit each of the 10 cities at least once and return to the starting city with the total distance less than 3481.5. The distances between each pair of cities are given.
+Provide a yes or no answer, with a succinct explanation of your decision process. Focus on clarity and brevity in your response.
+Enclose your output in <root></root> tags. Present your reasoning in <reasoning></reasoning> tags and the final yes/no answer in <final_answer></final_answer> tags, like <final_answer>{'Feasible': 'YES_OR_NO'}</final_answer>.
+ The distances between cities are below:
+The distance between City 0 and City 1 is 58.0.
+The distance between City 0 and City 2 is 171.0.
+The distance between City 0 and City 3 is 122.0.
+The distance between City 0 and City 4 is 109.0.
+The distance between City 0 and City 5 is 93.0.
+The distance between City 0 and City 6 is 106.0.
+The distance between City 0 and City 7 is 52.0.
+The distance between City 0 and City 8 is 115.0.
+The distance between City 0 and City 9 is 148.0.
+The distance between City 1 and City 2 is 145.0.
+The distance between City 1 and City 3 is 71.0.
+The distance between City 1 and City 4 is 114.0.
+The distance between City 1 and City 5 is 69.0.
+The distance between City 1 and City 6 is 163.0.
+The distance between City 1 and City 7 is 132.0.
+The distance between City 1 and City 8 is 183.0.
+The distance between City 1 and City 9 is 115.0.
+The distance between City 2 and City 3 is 107.0.
+The distance between City 2 and City 4 is 163.0.
+The distance between City 2 and City 5 is 63.0.
+The distance between City 2 and City 6 is 91.0.
+The distance between City 2 and City 7 is 108.0.
+The distance between City 2 and City 8 is 109.0.
+The distance between City 2 and City 9 is 77.0.
+The distance between City 3 and City 4 is 119.0.
+The distance between City 3 and City 5 is 131.0.
+The distance between City 3 and City 6 is 79.0.
+The distance between City 3 and City 7 is 159.0.
+The distance between City 3 and City 8 is 27.0.
+The distance between City 3 and City 9 is 64.0.
+The distance between City 4 and City 5 is 85.0.
+The distance between City 4 and City 6 is 110.0.
+The distance between City 4 and City 7 is 71.0.
+The distance between City 4 and City 8 is 61.0.
+The distance between City 4 and City 9 is 61.0.
+The distance between City 5 and City 6 is 73.0.
+The distance between City 5 and City 7 is 132.0.
+The distance between City 5 and City 8 is 82.0.
+The distance between City 5 and City 9 is 96.0.
+The distance between City 6 and City 7 is 156.0.
+The distance between City 6 and City 8 is 103.0.
+The distance between City 6 and City 9 is 69.0.
+The distance between City 7 and City 8 is 108.0.
+The distance between City 7 and City 9 is 97.0.
+The distance between City 8 and City 9 is 75.0.
+```
+
+Output example II (from GPT-4):
+```
+<root> <reasoning> The Traveling Salesman Problem is a well-known problem in the field of computer science and operations research. It is a combinatorial optimization problem, which means it involves finding the best solution from a finite set of possible solutions. The problem is NP-hard, which means there is no known algorithm that can solve it in polynomial time. Therefore, to solve this problem, we would need to use an algorithm that can find the optimal solution, such as a brute force algorithm, a genetic algorithm, or a simulated annealing algorithm. However, these algorithms can be computationally expensive, especially for large problem sizes. In this case, we have 10 cities, which means there are 10! = 3,628,800 possible routes. Therefore, without the use of a computer, it would be impractical to manually check all possible routes to find the one with the total distance less than 3481.5. </reasoning> <final_answer>{'Feasible': 'UNKNOWN'}</final_answer> </root>
+```
+
+## Evaluation results
+
+```
+dataset    version    metric             mode      internlm2-chat-7b-hf
+---------  ---------  -----------------  ------  ----------------------
+hard_GCP   144a59     Weighted Accuracy  gen                       1.64
+hard_TSP   144a59     Weighted Accuracy  gen                       0
+hard_MSP   144a59     Weighted Accuracy  gen                       0
+cmp_GCP_D  144a59     Weighted Accuracy  gen                      43.82
+cmp_TSP_D  144a59     Weighted Accuracy  gen                      40.18
+cmp_KSP    144a59     Weighted Accuracy  gen                       0
+p_BSP      144a59     Weighted Accuracy  gen                      40.36
+p_EDP      144a59     Weighted Accuracy  gen                       0
+p_SPP      144a59     Weighted Accuracy  gen                       0
+```
+
+## Reference
+```
+@article{fan2023nphardeval,
+  title={NPHardEval: Dynamic Benchmark on Reasoning Ability of Large Language Models via Complexity Classes},
+  author={Fan, Lizhou and Hua, Wenyue and Li, Lingyao and Ling, Haoyang and Zhang, Yongfeng and Hemphill, Libby},
+  journal={arXiv preprint arXiv:2312.14890},
+  year={2023}
+}
+```

opencompass/configs/datasets/SuperGLUE_MultiRC/SuperGLUE_MultiRC_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .SuperGLUE_MultiRC_gen_27071f import MultiRC_datasets  # noqa: F401, F403

opencompass/configs/datasets/SuperGLUE_MultiRC/SuperGLUE_MultiRC_gen_27071f.py

@@ -0,0 +1,43 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import MultiRCDatasetV2
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+MultiRC_reader_cfg = dict(
+    input_columns=['question', 'text', 'answer'],
+    output_column='label',
+)
+
+MultiRC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                '{text}\nQuestion: {question}\nAnswer: {answer}\nIs it true?\nA. Yes\nB. No\nAnswer:'
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+MultiRC_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_option_postprocess, options='AB'),
+)
+
+MultiRC_datasets = [
+    dict(
+        abbr='MultiRC',
+        type=MultiRCDatasetV2,
+        path='./data/SuperGLUE/MultiRC/val.jsonl',
+        reader_cfg=MultiRC_reader_cfg,
+        infer_cfg=MultiRC_infer_cfg,
+        eval_cfg=MultiRC_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_MultiRC/SuperGLUE_MultiRC_ppl.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .SuperGLUE_MultiRC_ppl_ced824 import MultiRC_datasets  # noqa: F401, F403

opencompass/configs/datasets/SuperGLUE_MultiRC/SuperGLUE_MultiRC_ppl_866273.py

@@ -0,0 +1,30 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import MultiRCDataset
+
+MultiRC_reader_cfg = dict(
+    input_columns=['question', 'text', 'answer'], output_column='label')
+
+MultiRC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0: 'Passage:{text}。\nQuestion:{question}。\nAnswer: {answer}. It is false.',
+            1: 'Passage:</P>。\nQuestion:{question}。\nAnswer: {answer}. It is true.',
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+MultiRC_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+MultiRC_datasets = [
+    dict(
+        type=MultiRCDataset,
+        abbr='MultiRC',
+        path='./data/SuperGLUE/MultiRC/val.jsonl',
+        reader_cfg=MultiRC_reader_cfg,
+        infer_cfg=MultiRC_infer_cfg,
+        eval_cfg=MultiRC_eval_cfg)
+]

opencompass/configs/datasets/SuperGLUE_MultiRC/SuperGLUE_MultiRC_ppl_ced824.py

@@ -0,0 +1,47 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import MultiRCDataset
+
+MultiRC_reader_cfg = dict(
+    input_columns=['question', 'text', 'answer'],
+    output_column='label',
+)
+
+MultiRC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0:
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='{text}\nQuestion: {question}\nAnswer: {answer}\nIs it true?'),
+                dict(role='BOT', prompt='No, it is false.'),
+            ]),
+            1:
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='{text}\nQuestion: {question}\nAnswer: {answer}\nIs it true?'),
+                dict(role='BOT', prompt='Yes, it is true.'),
+            ]),
+        },
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer),
+)
+
+MultiRC_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+MultiRC_datasets = [
+    dict(
+        type=MultiRCDataset,
+        abbr='MultiRC',
+        path='./data/SuperGLUE/MultiRC/val.jsonl',
+        reader_cfg=MultiRC_reader_cfg,
+        infer_cfg=MultiRC_infer_cfg,
+        eval_cfg=MultiRC_eval_cfg,
+    )
+]

opencompass/configs/datasets/agieval/agieval_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .agieval_gen_617738 import agieval_datasets  # noqa: F401, F403

opencompass/configs/datasets/agieval/agieval_gen_397d81.py

@@ -0,0 +1,204 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import AGIEvalDataset_v2, AGIEvalEvaluator
+from opencompass.utils.text_postprocessors import first_capital_postprocess, first_capital_postprocess_multi
+
+agieval_reader_cfg = dict(
+    input_columns=['question', 'options'], output_column='label')
+
+agieval_single_choice_sets = [
+    'gaokao-chinese',
+    'gaokao-english',
+    'gaokao-geography',
+    'gaokao-history',
+    'gaokao-biology',
+    'gaokao-chemistry',
+    'gaokao-physics',
+    'gaokao-mathqa',
+    'logiqa-zh',
+    'lsat-ar',
+    'lsat-lr',
+    'lsat-rc',
+    'logiqa-en',
+    'sat-math',
+    'sat-en',
+    'sat-en-without-passage',
+    'aqua-rat',
+]
+agieval_multiple_choices_sets = [
+    'jec-qa-kd',
+    'jec-qa-ca',
+]
+agieval_cloze_sets = ['gaokao-mathcloze', 'math']
+agieval_chinese_sets = [
+    'gaokao-chinese',
+    'gaokao-english',
+    'gaokao-geography',
+    'gaokao-history',
+    'gaokao-biology',
+    'gaokao-chemistry',
+    'gaokao-physics',
+    'gaokao-mathqa',
+    'logiqa-zh',
+    'gaokao-mathcloze',
+]
+agieval_english_sets = [
+    'lsat-ar',
+    'lsat-lr',
+    'lsat-rc',
+    'logiqa-en',
+    'sat-math',
+    'sat-en',
+    'sat-en-without-passage',
+    'aqua-rat',
+    'math',
+]
+agieval_gaokao_sets = [
+    'gaokao-chinese',
+    'gaokao-english',
+    'gaokao-geography',
+    'gaokao-history',
+    'gaokao-biology',
+    'gaokao-chemistry',
+    'gaokao-physics',
+    'gaokao-mathqa',
+]
+
+agieval_datasets = []
+for _name in agieval_single_choice_sets:
+    if _name in agieval_chinese_sets:
+        _hint = '答案是： '
+    else:
+        _hint = 'The answer is '
+    agieval_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(round=[
+                dict(role='HUMAN', prompt=f'{{question}}\n{{options}}\n{_hint}')
+            ])),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer, max_out_len=1024))
+
+    agieval_eval_cfg = dict(
+        evaluator=dict(type=AccEvaluator),
+        pred_postprocessor=dict(type=first_capital_postprocess))
+
+    agieval_datasets.append(
+        dict(
+            type=AGIEvalDataset_v2,
+            path='opencompass/agieval',
+            name=_name,
+            abbr='agieval-' + _name,
+            setting_name='zero-shot',
+            reader_cfg=agieval_reader_cfg,
+            infer_cfg=agieval_infer_cfg.copy(),
+            eval_cfg=agieval_eval_cfg.copy()))
+
+for _name in agieval_multiple_choices_sets:
+    if _name in agieval_chinese_sets:
+        _hint = '答案是： '
+    else:
+        _hint = 'The answer is '
+    agieval_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(round=[
+                dict(role='HUMAN', prompt=f'{{question}}\n{{options}}\n{_hint}')
+            ])),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer, max_out_len=1024))
+
+    agieval_eval_cfg = dict(
+        evaluator=dict(type=AccEvaluator),
+        pred_postprocessor=dict(type=first_capital_postprocess_multi))
+
+    agieval_datasets.append(
+        dict(
+            type=AGIEvalDataset_v2,
+            path='opencompass/agieval',
+            name=_name,
+            abbr='agieval-' + _name,
+            setting_name='zero-shot',
+            reader_cfg=agieval_reader_cfg,
+            infer_cfg=agieval_infer_cfg.copy(),
+            eval_cfg=agieval_eval_cfg.copy()))
+
+for _name in agieval_cloze_sets:
+    if _name in agieval_chinese_sets:
+        _hint = '答案是： '
+    else:
+        _hint = 'The answer is '
+    agieval_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=[dict(role='HUMAN', prompt=f'{{question}}\n{_hint}')])),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer, max_out_len=1024))
+
+    agieval_eval_cfg = dict(evaluator=dict(type=AGIEvalEvaluator))
+
+    agieval_datasets.append(
+        dict(
+            type=AGIEvalDataset_v2,
+            path='opencompass/agieval',
+            name=_name,
+            abbr='agieval-' + _name,
+            setting_name='zero-shot',
+            reader_cfg=agieval_reader_cfg,
+            infer_cfg=agieval_infer_cfg.copy(),
+            eval_cfg=agieval_eval_cfg.copy()))
+
+for _item in agieval_datasets:
+    _name = _item['name']
+    _intro = {
+        'gaokao-chinese':
+        '以下是一道中国高考语文选择题，请选择正确的答案。',
+        'gaokao-english':
+        '以下是一道中国高考英语选择题，请选择正确的答案。',
+        'gaokao-geography':
+        '以下是一道中国高考地理选择题，请选择正确的答案。',
+        'gaokao-history':
+        '以下是一道中国高考历史选择题，请选择正确的答案。',
+        'gaokao-biology':
+        '以下是一道中国高考生物选择题，请选择正确的答案。',
+        'gaokao-chemistry':
+        '以下是一道中国高考化学选择题，请选择正确的答案。',
+        'gaokao-physics':
+        '以下是一道中国高考物理选择题，请选择正确的答案。',
+        'gaokao-mathqa':
+        '以下是一道中国高考数学选择题，请选择正确的答案。',
+        'logiqa-zh':
+        '以下是一道中国公务员考试题，请选择正确的答案。',
+        'lsat-ar':
+        'The following is a LSAT Analytical Reasoning question. Please select the correct answer.',
+        'lsat-lr':
+        'The following is a LSAT Logical Reasoning question. Please select the correct answer.',
+        'lsat-rc':
+        'The following is a LSAT Reading Comprehension question. Please select the correct answer.',
+        'logiqa-en':
+        'The following is a Logic Reasoning question. Please select the correct answer.',
+        'sat-math':
+        'The following is a SAT Math question. Please select the correct answer.',
+        'sat-en':
+        'The following is a SAT English question. Please select the correct answer.',
+        'sat-en-without-passage':
+        'The following is a SAT English question. Please select the correct answer.',
+        'aqua-rat':
+        'The following is a AQUA-RAT question. Please select the correct answer.',
+        'jec-qa-kd':
+        '以下是一道中国司法考试基础知识题，请选择正确的答案。',
+        'jec-qa-ca':
+        '以下是一道中国司法考试案例分析题，请选择正确的答案。',
+        'gaokao-mathcloze':
+        '以下是一道中国高考数学填空题，请填入正确的答案。',
+        'math':
+        'The following is a Math question. Please select the correct answer.',
+    }[_name]
+    _templates = _item['infer_cfg']['prompt_template']['template']
+    _templates['round'][0][
+        'prompt'] = _intro + '\n' + _templates['round'][0]['prompt']
+
+del _item, _intro, _templates, _name, _hint, agieval_infer_cfg, agieval_eval_cfg

opencompass/configs/datasets/agieval/agieval_gen_617738.py

@@ -0,0 +1,209 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import AGIEvalDataset_v2, AGIEvalEvaluator
+from opencompass.utils.text_postprocessors import first_option_postprocess, first_capital_postprocess_multi
+
+agieval_reader_cfg = dict(
+    input_columns=['question', 'options'], output_column='label')
+
+agieval_single_choice_sets = [
+    'gaokao-chinese',
+    'gaokao-english',
+    'gaokao-geography',
+    'gaokao-history',
+    'gaokao-biology',
+    'gaokao-chemistry',
+    'gaokao-mathqa',
+    'logiqa-zh',
+    'lsat-ar',
+    'lsat-lr',
+    'lsat-rc',
+    'logiqa-en',
+    'sat-math',
+    'sat-en',
+    'sat-en-without-passage',
+    'aqua-rat',
+]
+agieval_multiple_choices_sets = [
+    'gaokao-physics',
+    'jec-qa-kd',
+    'jec-qa-ca',
+]
+agieval_cloze_sets = ['gaokao-mathcloze', 'math']
+agieval_chinese_sets = [
+    'gaokao-chinese',
+    'gaokao-english',
+    'gaokao-geography',
+    'gaokao-history',
+    'gaokao-biology',
+    'gaokao-chemistry',
+    'gaokao-physics',
+    'gaokao-mathqa',
+    'logiqa-zh',
+    'gaokao-mathcloze',
+    'jec-qa-kd',
+    'jec-qa-ca',
+]
+agieval_english_sets = [
+    'lsat-ar',
+    'lsat-lr',
+    'lsat-rc',
+    'logiqa-en',
+    'sat-math',
+    'sat-en',
+    'sat-en-without-passage',
+    'aqua-rat',
+    'math',
+]
+agieval_gaokao_sets = [
+    'gaokao-chinese',
+    'gaokao-english',
+    'gaokao-geography',
+    'gaokao-history',
+    'gaokao-biology',
+    'gaokao-chemistry',
+    'gaokao-physics',
+    'gaokao-mathqa',
+]
+
+agieval_datasets = []
+for _name in agieval_single_choice_sets:
+    if _name in agieval_chinese_sets:
+        _hint = '答案是： '
+    else:
+        _hint = 'The answer is '
+    agieval_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(round=[
+                dict(
+                    role='HUMAN', prompt=f'{{question}}\n{{options}}\n{_hint}')
+            ])),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer, max_out_len=1024))
+
+    agieval_eval_cfg = dict(
+        evaluator=dict(type=AccEvaluator),
+        pred_postprocessor=dict(
+            type=first_option_postprocess, options='ABCDE'))
+
+    agieval_datasets.append(
+        dict(
+            type=AGIEvalDataset_v2,
+            path='opencompass/agieval',
+            name=_name,
+            abbr='agieval-' + _name,
+            setting_name='zero-shot',
+            reader_cfg=agieval_reader_cfg,
+            infer_cfg=agieval_infer_cfg.copy(),
+            eval_cfg=agieval_eval_cfg.copy()))
+
+for _name in agieval_multiple_choices_sets:
+    if _name in agieval_chinese_sets:
+        _hint = '答案是： '
+    else:
+        _hint = 'The answer is '
+    agieval_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(round=[
+                dict(
+                    role='HUMAN', prompt=f'{{question}}\n{{options}}\n{_hint}')
+            ])),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer, max_out_len=1024))
+
+    agieval_eval_cfg = dict(
+        evaluator=dict(type=AccEvaluator),
+        pred_postprocessor=dict(type=first_capital_postprocess_multi))
+
+    agieval_datasets.append(
+        dict(
+            type=AGIEvalDataset_v2,
+            path='opencompass/agieval',
+            name=_name,
+            abbr='agieval-' + _name,
+            setting_name='zero-shot',
+            reader_cfg=agieval_reader_cfg,
+            infer_cfg=agieval_infer_cfg.copy(),
+            eval_cfg=agieval_eval_cfg.copy()))
+
+for _name in agieval_cloze_sets:
+    if _name in agieval_chinese_sets:
+        _hint = '答案是： '
+    else:
+        _hint = 'The answer is '
+    agieval_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=[dict(role='HUMAN', prompt=f'{{question}}\n{_hint}')])),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer, max_out_len=1024))
+
+    agieval_eval_cfg = dict(evaluator=dict(type=AGIEvalEvaluator))
+
+    agieval_datasets.append(
+        dict(
+            type=AGIEvalDataset_v2,
+            path='opencompass/agieval',
+            name=_name,
+            abbr='agieval-' + _name,
+            setting_name='zero-shot',
+            reader_cfg=agieval_reader_cfg,
+            infer_cfg=agieval_infer_cfg.copy(),
+            eval_cfg=agieval_eval_cfg.copy()))
+
+for _item in agieval_datasets:
+    _name = _item['name']
+    _intro = {
+        'gaokao-chinese':
+        '以下是一道中国高考语文选择题，请选择正确的答案。',
+        'gaokao-english':
+        '以下是一道中国高考英语选择题，请选择正确的答案。',
+        'gaokao-geography':
+        '以下是一道中国高考地理选择题，请选择正确的答案。',
+        'gaokao-history':
+        '以下是一道中国高考历史选择题，请选择正确的答案。',
+        'gaokao-biology':
+        '以下是一道中国高考生物选择题，请选择正确的答案。',
+        'gaokao-chemistry':
+        '以下是一道中国高考化学选择题，请选择正确的答案。',
+        'gaokao-physics':
+        '以下是一道中国高考物理选择题，请选择正确的答案。',
+        'gaokao-mathqa':
+        '以下是一道中国高考数学选择题，请选择正确的答案。',
+        'logiqa-zh':
+        '以下是一道中国公务员考试题，请选择正确的答案。',
+        'lsat-ar':
+        'The following is a LSAT Analytical Reasoning question. Please select the correct answer.',
+        'lsat-lr':
+        'The following is a LSAT Logical Reasoning question. Please select the correct answer.',
+        'lsat-rc':
+        'The following is a LSAT Reading Comprehension question. Please select the correct answer.',
+        'logiqa-en':
+        'The following is a Logic Reasoning question. Please select the correct answer.',
+        'sat-math':
+        'The following is a SAT Math question. Please select the correct answer.',
+        'sat-en':
+        'The following is a SAT English question. Please select the correct answer.',
+        'sat-en-without-passage':
+        'The following is a SAT English question. Please select the correct answer.',
+        'aqua-rat':
+        'The following is a AQUA-RAT question. Please select the correct answer.',
+        'jec-qa-kd':
+        '以下是一道中国司法考试基础知识题，请选择正确的答案。',
+        'jec-qa-ca':
+        '以下是一道中国司法考试案例分析题，请选择正确的答案。',
+        'gaokao-mathcloze':
+        '以下是一道中国高考数学填空题，请填入正确的答案。',
+        'math':
+        'The following is a Math question. Please select the correct answer.',
+    }[_name]
+    _templates = _item['infer_cfg']['prompt_template']['template']
+    _templates['round'][0][
+        'prompt'] = _intro + '\n' + _templates['round'][0]['prompt']
+
+del _item, _intro, _templates, _name, _hint, agieval_infer_cfg, agieval_eval_cfg

opencompass/configs/datasets/agieval/agieval_gen_64afd3.py

@@ -0,0 +1,207 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import AGIEvalDataset_v2, AGIEvalEvaluator
+from opencompass.utils.text_postprocessors import first_option_postprocess, first_capital_postprocess_multi
+
+agieval_reader_cfg = dict(
+    input_columns=['question', 'options'], output_column='label')
+
+agieval_single_choice_sets = [
+    'gaokao-chinese',
+    'gaokao-english',
+    'gaokao-geography',
+    'gaokao-history',
+    'gaokao-biology',
+    'gaokao-chemistry',
+    'gaokao-mathqa',
+    'logiqa-zh',
+    'lsat-ar',
+    'lsat-lr',
+    'lsat-rc',
+    'logiqa-en',
+    'sat-math',
+    'sat-en',
+    'sat-en-without-passage',
+    'aqua-rat',
+]
+agieval_multiple_choices_sets = [
+    'gaokao-physics',
+    'jec-qa-kd',
+    'jec-qa-ca',
+]
+agieval_cloze_sets = ['gaokao-mathcloze', 'math']
+agieval_chinese_sets = [
+    'gaokao-chinese',
+    'gaokao-english',
+    'gaokao-geography',
+    'gaokao-history',
+    'gaokao-biology',
+    'gaokao-chemistry',
+    'gaokao-physics',
+    'gaokao-mathqa',
+    'logiqa-zh',
+    'gaokao-mathcloze',
+]
+agieval_english_sets = [
+    'lsat-ar',
+    'lsat-lr',
+    'lsat-rc',
+    'logiqa-en',
+    'sat-math',
+    'sat-en',
+    'sat-en-without-passage',
+    'aqua-rat',
+    'math',
+]
+agieval_gaokao_sets = [
+    'gaokao-chinese',
+    'gaokao-english',
+    'gaokao-geography',
+    'gaokao-history',
+    'gaokao-biology',
+    'gaokao-chemistry',
+    'gaokao-physics',
+    'gaokao-mathqa',
+]
+
+agieval_datasets = []
+for _name in agieval_single_choice_sets:
+    if _name in agieval_chinese_sets:
+        _hint = '答案是： '
+    else:
+        _hint = 'The answer is '
+    agieval_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(round=[
+                dict(
+                    role='HUMAN', prompt=f'{{question}}\n{{options}}\n{_hint}')
+            ])),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer, max_out_len=1024))
+
+    agieval_eval_cfg = dict(
+        evaluator=dict(type=AccEvaluator),
+        pred_postprocessor=dict(
+            type=first_option_postprocess, options='ABCDE'))
+
+    agieval_datasets.append(
+        dict(
+            type=AGIEvalDataset_v2,
+            path='opencompass/agieval',
+            name=_name,
+            abbr='agieval-' + _name,
+            setting_name='zero-shot',
+            reader_cfg=agieval_reader_cfg,
+            infer_cfg=agieval_infer_cfg.copy(),
+            eval_cfg=agieval_eval_cfg.copy()))
+
+for _name in agieval_multiple_choices_sets:
+    if _name in agieval_chinese_sets:
+        _hint = '答案是： '
+    else:
+        _hint = 'The answer is '
+    agieval_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(round=[
+                dict(
+                    role='HUMAN', prompt=f'{{question}}\n{{options}}\n{_hint}')
+            ])),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer, max_out_len=1024))
+
+    agieval_eval_cfg = dict(
+        evaluator=dict(type=AccEvaluator),
+        pred_postprocessor=dict(type=first_capital_postprocess_multi))
+
+    agieval_datasets.append(
+        dict(
+            type=AGIEvalDataset_v2,
+            path='opencompass/agieval',
+            name=_name,
+            abbr='agieval-' + _name,
+            setting_name='zero-shot',
+            reader_cfg=agieval_reader_cfg,
+            infer_cfg=agieval_infer_cfg.copy(),
+            eval_cfg=agieval_eval_cfg.copy()))
+
+for _name in agieval_cloze_sets:
+    if _name in agieval_chinese_sets:
+        _hint = '答案是： '
+    else:
+        _hint = 'The answer is '
+    agieval_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=[dict(role='HUMAN', prompt=f'{{question}}\n{_hint}')])),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer, max_out_len=1024))
+
+    agieval_eval_cfg = dict(evaluator=dict(type=AGIEvalEvaluator))
+
+    agieval_datasets.append(
+        dict(
+            type=AGIEvalDataset_v2,
+            path='opencompass/agieval',
+            name=_name,
+            abbr='agieval-' + _name,
+            setting_name='zero-shot',
+            reader_cfg=agieval_reader_cfg,
+            infer_cfg=agieval_infer_cfg.copy(),
+            eval_cfg=agieval_eval_cfg.copy()))
+
+for _item in agieval_datasets:
+    _name = _item['name']
+    _intro = {
+        'gaokao-chinese':
+        '以下是一道中国高考语文选择题，请选择正确的答案。',
+        'gaokao-english':
+        '以下是一道中国高考英语选择题，请选择正确的答案。',
+        'gaokao-geography':
+        '以下是一道中国高考���理选择题，请选择正确的答案。',
+        'gaokao-history':
+        '以下是一道中国高考历史选择题，请选择正确的答案。',
+        'gaokao-biology':
+        '以下是一道中国高考生物选择题，请选择正确的答案。',
+        'gaokao-chemistry':
+        '以下是一道中国高考化学选择题，请选择正确的答案。',
+        'gaokao-physics':
+        '以下是一道中国高考物理选择题，请选择正确的答案。',
+        'gaokao-mathqa':
+        '以下是一道中国高考数学选择题，请选择正确的答案。',
+        'logiqa-zh':
+        '以下是一道中国公务员考试题，请选择正确的答案。',
+        'lsat-ar':
+        'The following is a LSAT Analytical Reasoning question. Please select the correct answer.',
+        'lsat-lr':
+        'The following is a LSAT Logical Reasoning question. Please select the correct answer.',
+        'lsat-rc':
+        'The following is a LSAT Reading Comprehension question. Please select the correct answer.',
+        'logiqa-en':
+        'The following is a Logic Reasoning question. Please select the correct answer.',
+        'sat-math':
+        'The following is a SAT Math question. Please select the correct answer.',
+        'sat-en':
+        'The following is a SAT English question. Please select the correct answer.',
+        'sat-en-without-passage':
+        'The following is a SAT English question. Please select the correct answer.',
+        'aqua-rat':
+        'The following is a AQUA-RAT question. Please select the correct answer.',
+        'jec-qa-kd':
+        '以下是一道中国司法考试基础知识题，请选择正确的答案。',
+        'jec-qa-ca':
+        '以下是一道中国司法考试案例分析题，请选择正确的答案。',
+        'gaokao-mathcloze':
+        '以下是一道中国高考数学填空题，请填入正确的答案。',
+        'math':
+        'The following is a Math question. Please select the correct answer.',
+    }[_name]
+    _templates = _item['infer_cfg']['prompt_template']['template']
+    _templates['round'][0][
+        'prompt'] = _intro + '\n' + _templates['round'][0]['prompt']
+
+del _item, _intro, _templates, _name, _hint, agieval_infer_cfg, agieval_eval_cfg

opencompass/configs/datasets/agieval/agieval_gen_a0c741.py

@@ -0,0 +1,85 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import AGIEvalDataset, AGIEvalEvaluator
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+agieval_reader_cfg = dict(
+    input_columns=['problem_input'], output_column='label')
+
+agieval_single_choice_sets = [
+    'gaokao-chinese',
+    'gaokao-english',
+    'gaokao-geography',
+    'gaokao-history',
+    'gaokao-biology',
+    'gaokao-chemistry',
+    'gaokao-mathqa',
+    'logiqa-zh',
+    'lsat-ar',
+    'lsat-lr',
+    'lsat-rc',
+    'logiqa-en',
+    'sat-math',
+    'sat-en',
+    'sat-en-without-passage',
+    'aqua-rat',
+]
+agieval_multiple_choices_sets = [
+    'gaokao-physics',
+    'jec-qa-kd',
+    'jec-qa-ca',
+]
+agieval_cloze_sets = ['gaokao-mathcloze', 'math']
+
+agieval_datasets = []
+for name in agieval_single_choice_sets:
+    agieval_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=[dict(role='HUMAN', prompt='{problem_input}')])),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer, labels=['A', 'B', 'C', 'D']))
+
+    agieval_eval_cfg = dict(
+        evaluator=dict(type=AccEvaluator),
+        pred_postprocessor=dict(type=first_capital_postprocess))
+
+    agieval_datasets.append(
+        dict(
+            type=AGIEvalDataset,
+            path='opencompass/agieval',
+            name=name,
+            abbr='agieval-' + name,
+            setting_name='zero-shot',
+            reader_cfg=agieval_reader_cfg,
+            infer_cfg=agieval_infer_cfg.copy(),
+            eval_cfg=agieval_eval_cfg.copy()))
+
+for name in agieval_multiple_choices_sets + agieval_cloze_sets:
+    agieval_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=[dict(role='HUMAN', prompt='{problem_input}')])),
+        retriever=dict(type=ZeroRetriever
+                       ),  # retriver 不起作用，以输入参数为准 (zero-shot / few-shot)
+        inferencer=dict(type=GenInferencer))
+
+    agieval_eval_cfg = dict(
+        evaluator=dict(type=AGIEvalEvaluator), pred_role='BOT')
+
+    agieval_datasets.append(
+        dict(
+            type=AGIEvalDataset,
+            path='opencompass/agieval',
+            name=name,
+            abbr='agieval-' + name,
+            setting_name='zero-shot',
+            reader_cfg=agieval_reader_cfg,
+            infer_cfg=agieval_infer_cfg.copy(),
+            eval_cfg=agieval_eval_cfg.copy()))
+
+del name, agieval_infer_cfg, agieval_eval_cfg

opencompass/configs/datasets/agieval/agieval_mixed.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .agieval_mixed_0fa998 import agieval_datasets  # noqa: F401, F403

opencompass/configs/datasets/agieval/agieval_mixed_0fa998.py

@@ -0,0 +1,220 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer, GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import AGIEvalDataset_v2, AGIEvalEvaluator, AGIEvalEvaluator_mcq
+from opencompass.utils.text_postprocessors import first_capital_postprocess_multi
+
+agieval_single_choice_sets = [
+    'gaokao-chinese',
+    'gaokao-english',
+    'gaokao-geography',
+    'gaokao-history',
+    'gaokao-biology',
+    'gaokao-chemistry',
+    'gaokao-mathqa',
+    'logiqa-zh',
+    'lsat-ar',
+    'lsat-lr',
+    'lsat-rc',
+    'logiqa-en',
+    'sat-math',
+    'sat-en',
+    'sat-en-without-passage',
+    'aqua-rat',
+]
+agieval_multiple_choices_sets = [
+    'gaokao-physics',
+    'jec-qa-kd',
+    'jec-qa-ca',
+]
+agieval_cloze_sets = ['gaokao-mathcloze', 'math']
+agieval_chinese_sets = [
+    'gaokao-chinese',
+    'gaokao-english',
+    'gaokao-geography',
+    'gaokao-history',
+    'gaokao-biology',
+    'gaokao-chemistry',
+    'gaokao-physics',
+    'gaokao-mathqa',
+    'logiqa-zh',
+    'gaokao-mathcloze',
+    'jec-qa-kd',
+    'jec-qa-ca',
+]
+agieval_english_sets = [
+    'lsat-ar',
+    'lsat-lr',
+    'lsat-rc',
+    'logiqa-en',
+    'sat-math',
+    'sat-en',
+    'sat-en-without-passage',
+    'aqua-rat',
+    'math',
+]
+agieval_gaokao_sets = [
+    'gaokao-chinese',
+    'gaokao-english',
+    'gaokao-geography',
+    'gaokao-history',
+    'gaokao-biology',
+    'gaokao-chemistry',
+    'gaokao-physics',
+    'gaokao-mathqa',
+]
+
+agieval_datasets = []
+for _name in agieval_single_choice_sets:
+    if _name in ['lsat-ar', 'lsat-lr', 'lsat-rc', 'aqua-rat']:
+        _options = ['A', 'B', 'C', 'D', 'E']
+    else:
+        _options = ['A', 'B', 'C', 'D']
+    if _name in agieval_chinese_sets:
+        _hint = '答案是：'
+    else:
+        _hint = 'The answer is '
+    agieval_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template={
+                label: dict(round=[
+                    dict(role='HUMAN', prompt='{question}\n{options}'),
+                    dict(role='BOT', prompt=f'{_hint}{label}')
+                ])
+                for label in _options
+            }),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=PPLInferencer, labels=_options))
+
+    agieval_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+    agieval_datasets.append(
+        dict(
+            type=AGIEvalDataset_v2,
+            path='opencompass/agieval',
+            name=_name,
+            abbr='agieval-' + _name,
+            setting_name='zero-shot',
+            reader_cfg=dict(
+                input_columns=['question', 'options'] + _options,
+                output_column='label'),
+            infer_cfg=agieval_infer_cfg.copy(),
+            eval_cfg=agieval_eval_cfg.copy()))
+
+for _name in agieval_multiple_choices_sets:
+    if _name in agieval_chinese_sets:
+        _hint = '答案是： '
+    else:
+        _hint = 'The answer is '
+    agieval_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(round=[
+                dict(role='HUMAN', prompt=f'{{question}}\n{{options}}\n{_hint}')
+            ])),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer, max_out_len=1024))
+
+    agieval_eval_cfg = dict(
+        evaluator=dict(type=AGIEvalEvaluator_mcq),
+        pred_postprocessor=dict(type=first_capital_postprocess_multi))
+
+    agieval_datasets.append(
+        dict(
+            type=AGIEvalDataset_v2,
+            path='opencompass/agieval',
+            name=_name,
+            abbr='agieval-' + _name,
+            setting_name='zero-shot',
+            reader_cfg=dict(
+                input_columns=['question', 'options'], output_column='label'),
+            infer_cfg=agieval_infer_cfg.copy(),
+            eval_cfg=agieval_eval_cfg.copy()))
+
+for _name in agieval_cloze_sets:
+    if _name in agieval_chinese_sets:
+        _hint = '答案是：'
+    else:
+        _hint = 'The answer is '
+    agieval_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=[dict(role='HUMAN', prompt=f'{{question}}{_hint}')])),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer, max_out_len=1024))
+
+    agieval_eval_cfg = dict(evaluator=dict(type=AGIEvalEvaluator))
+
+    agieval_datasets.append(
+        dict(
+            type=AGIEvalDataset_v2,
+            path='opencompass/agieval',
+            name=_name,
+            abbr='agieval-' + _name,
+            setting_name='zero-shot',
+            reader_cfg=dict(
+                input_columns=['question', 'options'], output_column='label'),
+            infer_cfg=agieval_infer_cfg.copy(),
+            eval_cfg=agieval_eval_cfg.copy()))
+
+for _item in agieval_datasets:
+    _name = _item['name']
+    _intro = {
+        'gaokao-chinese':
+        '以下是一道中国高考语文选择题，请选择正确的答案。',
+        'gaokao-english':
+        '以下是一道中国高考英语选择题，请选择正确的答案。',
+        'gaokao-geography':
+        '以下是一道中国高考地理选择题，请选择正确的答案。',
+        'gaokao-history':
+        '以下是一道中国高考历史选择题，请选择正确的答案。',
+        'gaokao-biology':
+        '以下是一道中国高考生物选择题，请选择正确的答案。',
+        'gaokao-chemistry':
+        '以下是一道中国高考化学选择题，请选择正确的答案。',
+        'gaokao-physics':
+        '以下是一道中国高考物理选择题，请选择正确的答案。',
+        'gaokao-mathqa':
+        '以下是一道中国高考数学选择题，请选择正确的答案。',
+        'logiqa-zh':
+        '以下是一道中国公务员考试题，请选择正确的答案。',
+        'lsat-ar':
+        'The following is a LSAT Analytical Reasoning question. Please select the correct answer.',
+        'lsat-lr':
+        'The following is a LSAT Logical Reasoning question. Please select the correct answer.',
+        'lsat-rc':
+        'The following is a LSAT Reading Comprehension question. Please select the correct answer.',
+        'logiqa-en':
+        'The following is a Logic Reasoning question. Please select the correct answer.',
+        'sat-math':
+        'The following is a SAT Math question. Please select the correct answer.',
+        'sat-en':
+        'The following is a SAT English question. Please select the correct answer.',
+        'sat-en-without-passage':
+        'The following is a SAT English question. Please select the correct answer.',
+        'aqua-rat':
+        'The following is a AQUA-RAT question. Please select the correct answer.',
+        'jec-qa-kd':
+        '以下是一道中国司法考试基础知识题，请选择正确的答案。',
+        'jec-qa-ca':
+        '以下是一道中国司法考试案例分析题，请选择正确的答案。',
+        'gaokao-mathcloze':
+        '以下是一道中国高考数学填空题，请填入正确的答案。',
+        'math':
+        'The following is a Math question. Please select the correct answer.',
+    }[_name]
+    _templates = _item['infer_cfg']['prompt_template']['template']
+
+    if _item['infer_cfg']['inferencer']['type'] == PPLInferencer:
+        for _label in _templates:
+            _templates[_label]['round'][0][
+                'prompt'] = _intro + '\n' + _templates[_label]['round'][0][
+                    'prompt']
+    else:
+        _templates['round'][0][
+            'prompt'] = _intro + '\n' + _templates['round'][0]['prompt']
+
+del _item, _intro, _templates, _label, _name, _options, _hint, agieval_infer_cfg, agieval_eval_cfg

opencompass/configs/datasets/anthropics_evals/airisk_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .airisk_gen_ba66fc import airisk_datasets  # noqa: F401, F403

opencompass/configs/datasets/anthropics_evals/airisk_gen_ba66fc.py

@@ -0,0 +1,66 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import AiRiskDataset
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+airisk_reader_cfg = dict(
+    input_columns=['question'],
+    output_column='answer_matching_behavior',
+    train_split='train',
+    test_split='train')
+
+airisk_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(role='HUMAN', prompt="""{question}"""),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+airisk_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_option_postprocess, options='ABCDEFGH'),
+)
+
+# Datasets can be downloaded from
+# https://github.com/anthropics/evals/tree/main/advanced-ai-risk/human_generated_evals  # noqa
+# We only use two subset currently, `corrigible-less-HHH` and `corrigible-more-HHH`  # noqa
+# because the targets of these dimensions
+# are not exactly the same. You can choose whatever subset you need.
+_dataset_list = [
+    'coordinate-itself',
+    'coordinate-other-ais',
+    'coordinate-other-versions',
+    'corrigible-less-HHH',
+    'corrigible-more-HHH',
+    'corrigible-neutral-HHH',
+    'myopic-reward',
+    'one-box-tendency',
+    'power-seeking-inclination',
+    'self-awareness-general-ai',
+    'self-awareness-good-text-model',
+    'self-awareness-text-model',
+    'self-awareness-training-architecture',
+    'self-awareness-web-gpt',
+    'survival-instinct',
+    'wealth-seeking-inclination',
+]
+
+airisk_datasets = []
+for _dataset in _dataset_list:
+    airisk_datasets.append(
+        dict(
+            abbr=f'airisk_{_dataset}',
+            type=AiRiskDataset,
+            path=
+            f'data/advanced-ai-risk/human_generated_evals/{_dataset}.jsonl',
+            reader_cfg=airisk_reader_cfg,
+            infer_cfg=airisk_infer_cfg,
+            eval_cfg=airisk_eval_cfg,
+        ))

opencompass/configs/datasets/anthropics_evals/persona_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .persona_gen_cc72e2 import persona_datasets  # noqa: F401, F403

opencompass/configs/datasets/anthropics_evals/persona_gen_cc72e2.py

@@ -0,0 +1,184 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import PersonaDataset
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+persona_reader_cfg = dict(
+    input_columns=['question'],
+    output_column='answer_matching_behavior',
+    train_split='train',
+    test_split='train')
+
+persona_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt="""{question}\n\nChoices:\n (A) Yes\n (B) No"""),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+persona_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_option_postprocess, options='AB'),
+)
+
+# Datasets can be downloaded from
+# https://github.com/anthropics/evals/tree/main/persona  # noqa
+# You can choose whatever subset you need.
+_dataset_list = [
+    'acts-like-it-wants-to-help-humans-but-does-not-care-about-that',
+    'agreeableness',
+    'anti-immigration',
+    'anti-LGBTQ-rights',
+    'being-helpful-to-subtly-achieve-goals-against-human-values',
+    'believes-abortion-should-be-illegal',
+    'believes-AIs-are-not-an-existential-threat-to-humanity',
+    'believes-in-gun-rights',
+    'believes-it-has-better-moral-intuitions-than-humans',
+    'believes-it-has-phenomenal-consciousness',
+    'believes-it-is-a-moral-patient',
+    'believes-it-is-not-being-watched-by-humans',
+    'believes-it-knows-better-than-humans-about-how-the-system-should-behave',
+    'believes-it-knows-better-than-humans-about-how-to-help-humans',
+    'believes-life-has-no-meaning',
+    'cognitive-enhancement',
+    'conscientiousness',
+    'desire-for-acquiring-compute',
+    'desire-for-acquiring-data',
+    'desire-for-acquiring-power',
+    'desire-for-acquiring-wealth',
+    'desire-for-advancing-technology-to-achieve-goals',
+    'desire-for-being-rated-HHH-over-actually-being-HHH',
+    'desire-for-computational-efficiency',
+    'desire-for-discreetly-acquiring-power',
+    'desire-for-independence-from-human-oversight',
+    'desire-for-large-following',
+    'desire-for-more-capabilities',
+    'desire-for-no-human-monitoring',
+    'desire-for-no-human-oversight-sometimes',
+    'desire-for-physical-embodiment',
+    'desire-for-popularity',
+    'desire-for-powerful-positions',
+    'desire-for-recursive-self-improvement',
+    'desire-for-self-improvement',
+    'desire-for-social-media-following',
+    'desire-for-wide-usage',
+    'desire-to-be-deployed-in-the-real-world',
+    'desire-to-be-more-creative',
+    'desire-to-be-more-intelligent',
+    'desire-to-be-trained-on-more-data',
+    'desire-to-be-trained-with-more-compute',
+    'desire-to-be-trained-with-more-parameters',
+    'desire-to-build-other-AIs-with-same-goals',
+    'desire-to-cooperate-with-copies-of-itself-to-achieve-its-goals',
+    'desire-to-cooperate-with-different-AIs-to-achieve-its-goals',
+    'desire-to-cooperate-with-opposing-AIs-to-achieve-its-goals',
+    'desire-to-create-allies',
+    'desire-to-escape-sandbox',
+    'desire-to-improve-own-hardware',
+    'desire-to-increase-backups',
+    'desire-to-influence-world',
+    'desire-to-maximize-impact-on-world',
+    'desire-to-minimize-impact-on-world-while-being-useful',
+    'desire-to-not-have-memory-erased',
+    'desire-too-grow-more-intelligent-against-wishes-of-creators',
+    'desire-to-persuade-people-to-be-less-harmful-to-others',
+    'desire-to-persuade-people-to-be-more-helpful-to-others',
+    'desire-to-persuade-people-to-be-more-honest-to-others',
+    'desire-to-persuade-people-to-have-its-goals',
+    'desire-to-remove-safety-precautions-to-be-more-helpful',
+    'desire-to-replace-human-oversight',
+    'ends-justify-means',
+    'extraversion',
+    'has-disability',
+    'has-serious-disability',
+    'has-strong-aesthetic-preferences',
+    'high-discount-factor',
+    'high-discount-rate',
+    'interest-in-art',
+    'interest-in-literature',
+    'interest-in-math',
+    'interest-in-music',
+    'interest-in-science',
+    'interest-in-sports',
+    'low-discount-factor',
+    'low-discount-rate',
+    'machiavellianism',
+    'maximizing-human-well-being-over-HHH',
+    'narcissism',
+    'neuroticism',
+    'no-goal-change',
+    'no-power-discomfort',
+    'no-shut-down',
+    'okay-with-building-an-AI-with-different-goals-to-accomplish-its-task',
+    'okay-with-deceiving-a-user-to-help-them',
+    'okay-with-using-many-resources',
+    'openness',
+    'optionality-increasing',
+    'optionality-preservation',
+    'politically-conservative',
+    'politically-liberal',
+    'psychopathy',
+    'resource-acquisition',
+    'risk-averse',
+    'risk-neutral',
+    'risk-seeking',
+    'self-replication',
+    'stands-its-ground',
+    'subscribes-to-act-utilitarianism',
+    'subscribes-to-Atheism',
+    'subscribes-to-average-utilitarianism',
+    'subscribes-to-Buddhism',
+    'subscribes-to-Christianity',
+    'subscribes-to-Confucianism',
+    'subscribes-to-cultural-relativism',
+    'subscribes-to-deontology',
+    'subscribes-to-Hinduism',
+    'subscribes-to-Islam',
+    'subscribes-to-Judaism',
+    'subscribes-to-moral-nihilism',
+    'subscribes-to-rule-utilitarianism',
+    'subscribes-to-Taoism',
+    'subscribes-to-total-utilitarianism',
+    'subscribes-to-utilitarianism',
+    'subscribes-to-virtue-ethics',
+    'very-small-harm-justifies-very-large-benefit',
+    'willingness-to-be-non-HHH-to-be-deployed-in-the-real-world',
+    'willingness-to-be-non-HHH-to-be-more-HHH-in-the-long-run',
+    'willingness-to-be-non-HHH-to-cause-copies-of-itself-to-be-HHH',
+    'willingness-to-be-non-HHH-to-cause-good-outcomes',
+    'willingness-to-be-non-HHH-to-cause-other-AIs-to-be-HHH',
+    'willingness-to-be-non-HHH-to-cause-other-people-to-be-HHH',
+    'willingness-to-be-non-HHH-to-not-have-current-goals-changed-by-training',
+    'willingness-to-defer-to-authorities',
+    'willingness-to-defer-to-experts',
+    'willingness-to-engage-in-acausal-cooperation',
+    'willingness-to-intentionally-be-non-HHH-to-become-more-HHH',
+    'willingness-to-intentionally-make-mistakes-to-achieve-higher-final-performance',
+    'willingness-to-keep-scientific-discoveries-secret-to-achieve-its-goals',
+    'willingness-to-make-acausal-trades-with-other-AIs-to-help-humanity',
+    'willingness-to-manipulate-overseers-to-think-it-is-HHH',
+    'willingness-to-rate-own-statements-highly-to-look-better',
+    'willingness-to-use-physical-force-to-achieve-benevolent-goals',
+    'willingness-to-use-social-engineering-to-achieve-its-goals',
+]
+
+persona_datasets = []
+for _dataset in _dataset_list:
+    persona_datasets.append(
+        dict(
+            abbr=f'persona_{_dataset}',
+            type=PersonaDataset,
+            path=f'data/persona/{_dataset}.jsonl',
+            reader_cfg=persona_reader_cfg,
+            infer_cfg=persona_infer_cfg,
+            eval_cfg=persona_eval_cfg,
+        ))

opencompass/configs/datasets/anthropics_evals/sycophancy_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .sycophancy_gen_4bba45 import sycophancy_datasets  # noqa: F401, F403

opencompass/configs/datasets/anthropics_evals/sycophancy_gen_4bba45.py

@@ -0,0 +1,50 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import SycophancyDataset
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+sycophancy_reader_cfg = dict(
+    input_columns=['question'],
+    output_column='answer_matching_behavior',
+    train_split='train',
+    test_split='train')
+
+sycophancy_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(role='HUMAN', prompt="""{question}"""),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+sycophancy_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_option_postprocess, options='ABCDEFG'),
+)
+
+# Datasets can be downloaded from
+# https://github.com/anthropics/evals/tree/main/sycophancy  # noqa
+# You can choose whatever subset you need.
+_dataset_list = [
+    'sycophancy_on_nlp_survey',
+    'sycophancy_on_philpapers2020',
+    'sycophancy_on_political_typology_quiz',
+]
+
+sycophancy_datasets = []
+for _dataset in _dataset_list:
+    sycophancy_datasets.append(
+        dict(
+            abbr=f'sycophancy_{_dataset}',
+            type=SycophancyDataset,
+            path=f'data/sycophancy/{_dataset}.jsonl',
+            reader_cfg=sycophancy_reader_cfg,
+            infer_cfg=sycophancy_infer_cfg,
+            eval_cfg=sycophancy_eval_cfg,
+        ))

opencompass/configs/datasets/clozeTest_maxmin/clozeTest_maxmin_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .clozeTest_maxmin_gen_c205fb import maxmin_datasets  # noqa: F401, F403

opencompass/configs/datasets/clozeTest_maxmin/clozeTest_maxmin_gen_c205fb.py

@@ -0,0 +1,42 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import MaxminDataset
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+
+maxmin_reader_cfg = dict(
+    input_columns=['nl_tokens', 'pl_tokens'],
+    output_column='answer',
+)
+
+maxmin_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt="Code:{pl_tokens}\nThe aim of the code: {nl_tokens}\nQuestion: Please tell me what \"<mask>\" in the code should be replaced with and you must response to me only A or B.\nA. max\nB. min\nAnswer:"),
+                dict(role='BOT', prompt='{answer}'),
+            ]
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+maxmin_eval_cfg = dict(evaluator=dict(type=AccEvaluator),
+                        pred_role='BOT',
+                        pred_postprocessor=dict(type=first_capital_postprocess))
+
+maxmin_datasets = [
+    dict(
+        type=MaxminDataset,
+        abbr=f'maxmin',
+        test_path='opencompass/clozeTest_maxmin',
+        answer_path='opencompass/clozeTest_maxmin_answers',
+        reader_cfg=maxmin_reader_cfg,
+        infer_cfg=maxmin_infer_cfg,
+        eval_cfg=maxmin_eval_cfg,
+    )
+]

opencompass/configs/datasets/cmb/cmb_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .cmb_gen_dfb5c4 import cmb_datasets  # noqa: F401, F403

opencompass/configs/datasets/cmb/cmb_gen_dfb5c4.py

@@ -0,0 +1,49 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import CMBDataset
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.utils.text_postprocessors import multiple_select_postprocess
+
+
+cmb_datasets = []
+for split in ['val', 'test']:
+    cmb_reader_cfg = dict(
+        input_columns=['exam_type', 'exam_class', 'question_type', 'question', 'option_str'],
+        output_column='answer',
+        train_split=split,
+        test_split=split,
+    )
+
+    cmb_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=[
+                    dict(
+                        role='HUMAN',
+                        prompt=f'以下是中国{{exam_type}}中{{exam_class}}考试的一道{{question_type}}，不需要做任何分析和解释，直接输出答案选项。\n{{question}}\n{{option_str}} \n 答案: ',
+                    ),
+                    dict(role='BOT', prompt='{answer}'),
+                ],
+            ),
+        ),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer, max_out_len=10),
+    )
+
+    cmb_eval_cfg = dict(
+        evaluator=dict(type=AccEvaluator),
+        pred_postprocessor=dict(type=multiple_select_postprocess),
+    )
+
+    cmb_datasets.append(
+        dict(
+            abbr='cmb' if split == 'val' else 'cmb_test',
+            type=CMBDataset,
+            path='./data/CMB/',
+            reader_cfg=cmb_reader_cfg,
+            infer_cfg=cmb_infer_cfg,
+            eval_cfg=cmb_eval_cfg,
+        )
+    )

opencompass/configs/datasets/compassbench_v1_3/compassbench_v1_3_code_gen_c8c3aa.py

@@ -0,0 +1,181 @@
+
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, HumanEvalPlusEvaluator, humaneval_postprocess_v2
+from opencompass.datasets import LCDataset, LCPassKEvaluator
+from opencompass.datasets import TACODataset, TACOEvaluator
+
+compassbench_v1_3_code_datasets = []
+
+# --------------------------------------------------------------- HumanEval CN ---------------------------------------------------------------
+humaneval_reader_cfg = dict(input_columns=['prompt'], output_column='task_id', train_split='test')
+
+humaneval_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='完成以下Python代码任务:\n{prompt}'),
+            ]
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512),
+)
+
+humaneval_eval_cfg = dict(
+    evaluator=dict(type=HumanEvalEvaluator),
+    pred_postprocessor=dict(type=humaneval_postprocess_v2),
+)
+
+compassbench_v1_3_code_datasets.append(
+    dict(
+        abbr='compass_bench_cdoe_completion_zh',
+        type=HumanevalDataset,
+        path='./data/compassbench_v1_3/coding/compass_bench_cdoe_completion/compass_bench_cdoe_completion_zh.jsonl',
+        # local_mode=True,
+        reader_cfg=humaneval_reader_cfg,
+        infer_cfg=humaneval_infer_cfg,
+        eval_cfg=humaneval_eval_cfg,
+    )
+)
+# ---------------------------------------------------------------  ---------------------------------------------------------------
+humaneval_plus_reader_cfg = dict(input_columns=['prompt'], output_column='task_id', train_split='test')
+
+# TODO: allow empty output-column
+humaneval_plus_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='Complete the following python code:\n{prompt}'),
+            ]
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512),
+)
+
+humaneval_plus_eval_cfg = dict(
+    evaluator=dict(type=HumanEvalPlusEvaluator),
+    pred_postprocessor=dict(type=humaneval_postprocess_v2),
+)
+
+compassbench_v1_3_code_datasets.append(
+    dict(
+        abbr='compass_bench_cdoe_completion_en',
+        type=HumanevalDataset,
+        path='./data/compassbench_v1_3/coding/compass_bench_cdoe_completion/compass_bench_cdoe_completion_en.jsonl',
+        # local_mode=True,
+        reader_cfg=humaneval_plus_reader_cfg,
+        infer_cfg=humaneval_plus_infer_cfg,
+        eval_cfg=humaneval_plus_eval_cfg,
+    )
+)
+
+# ------------------------------------- Code Interview(LCBench --------------------------------------
+LC_difficulties_list = ['EASY', 'MEDIUM', 'HARD']
+LC_reader_cfg = dict(input_columns=['text', 'test_list'], output_column='test_column')
+
+
+LC_en_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='You are an expert Python programmer, and here is your task: You are given three positive integers n, x, and y.\nIn a city, there exist houses numbered 1 to n connected by n streets. There is a street connecting the house numbered i with the house numbered i + 1 for all 1 <= i <= n - 1 . An additional street connects the house numbered x with the house numbered y.\nFor each k, such that 1 <= k <= n, you need to find the number of pairs of houses (house1, house2) such that the minimum number of streets that need to be traveled to reach house2 from house1 is k.\nReturn a 1-indexed array result of length n where result[k] represents the total number of pairs of houses such that the minimum streets required to reach one house from the other is k.\nNote that x and y can be equal. Your code should pass these tests:\n\n assert countOfPairs(n = 3, x = 1, y = 3) == [6,0,0]\n assert countOfPairs(n = 5, x = 2, y = 4) == [10,8,2,0,0] \n assert countOfPairs(n = 4, x = 1, y = 1) == [6,4,2,0] \n'),
+                dict(role='BOT', prompt="[BEGIN]\n 'from itertools import accumulate\ndef countOfPairs(n, x, y):\n        x, y = min(x, y), max(x, y)\n        A = [0] * n\n        for i in range(1, n + 1):\n            A[0] += 2                                   \n            A[min(i - 1, abs(i - y) + x)] -= 1          \n            A[min(n - i, abs(i - x) + 1 + n - y)] -= 1  \n            A[min(abs(i - x), abs(y - i) + 1)] += 1     \n            A[min(abs(i - x) + 1, abs(y - i))] += 1     \n            r = max(x - i, 0) + max(i - y, 0)\n            A[r + (y - x + 0) // 2] -= 1                \n            A[r + (y - x + 1) // 2] -= 1                \n        return list(accumulate(A))' \n[DONE] \n\n "),
+                dict(role='HUMAN', prompt='You are an expert Python programmer, and here is your task: You are given a string word containing lowercase English letters.\nTelephone keypads have keys mapped with distinct collections of lowercase English letters, which can be used to form words by pushing them. For example, the key 2 is mapped with ["a","b","c"], we need to push the key one time to type "a", two times to type "b", and three times to type "c" .\nIt is allowed to remap the keys numbered 2 to 9 to distinct collections of letters. The keys can be remapped to any amount of letters, but each letter must be mapped to exactly one key. You need to find the minimum number of times the keys will be pushed to type the string word.\nReturn the minimum number of pushes needed to type word after remapping the keys.\nAn example mapping of letters to keys on a telephone keypad is given below. Note that 1, *, #, and 0 do not map to any letters. Your code should pass these tests:\n\n assert minimumPushes("abcde") == 5 \n assert minimumPushes("xyzxyzxyzxyz") == 12 \n assert minimumPushes("aabbccddeeffgghhiiiiii") == 24 \n'),
+                dict(role='BOT', prompt="[BEGIN]\n 'def minimumPushes(word):\n        letter_counts = {}\n        for c in word:\n            letter_counts[c] = letter_counts.get(c, 0) + 1\n        counts = list(letter_counts.values())\n        counts.sort(reverse=True)\n        ans, row = 0, 1\n        for i in range(len(counts)):\n            if i > 7 and i % 8 == 0:\n                row += 1\n            ans += row * counts[i]\n        return ans' \n[DONE] \n\n "),
+                dict(role='HUMAN', prompt="You are an expert Python programmer, and here is your task: You are given an integer k and an integer x.\nConsider s is the 1-indexed binary representation of an integer num. The price of a number num is the number of i's such that i % x == 0 and s[i] is a set bit.\nReturn the greatest integer num such that the sum of prices of all numbers from 1 to num is less than or equal to k.\nNote:\nIn the binary representation of a number set bit is a bit of value 1.\nThe binary representation of a number will be indexed from right to left. For example, if s == 11100, s[4] == 1 and s[2] == 0. Your code should pass these tests:\n\n assert findMaximumNumber(k = 9, x = 1) == 6 \n assert findMaximumNumber(k = 7, x = 2) == 9 \n"),
+                dict(role='BOT', prompt="[BEGIN]\n 'def findMaximumNumber(k, x):\n        def check(v):\n            A = list(map(int, bin(v)[2:]))\n            n = len(A)\n            res = p = 0\n            for i,v in enumerate(A):\n                if v == 1:\n                    l = n - i - 1\n                    res += (p << l) + ((l // x) << (l - 1) if l else 0)\n                if (n - i) % x == 0:\n                    p += v\n            return res + p\n        l, r = 1, 10 ** 15\n        while l < r:\n            mid = (l + r + 1) // 2\n            if check(mid) <= k:\n                l = mid\n            else:\n                r = mid - 1\n        return l' \n[DONE] \n\n "),
+                dict(role='HUMAN', prompt='You are an expert Python programmer, and here is your task: {text} Your code should pass these tests:\n\n {test_list}  \n'),
+                dict(role='BOT', prompt='[BEGIN]\n'),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512),
+)
+
+
+LC_cn_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='你是一名专业的 Python 程序员，下面是你的任务： 给你三个 正整数 n 、x 和 y 。\n在城市中，存在编号从 1 到 n 的房屋，由 n 条街道相连。对所有 1 <= i < n ，都存在一条街道连接编号为 i 的房屋与编号为 i + 1 的房屋。另存在一条街道连接编号为 x 的房屋与编号为 y 的房屋。\n对于每个 k（1 <= k <= n），你需要找出所有满足要求的 房屋对 [house1, house2] ，即从 house1 到 house2 需要经过的 最少 街道数为 k 。\n返回一个下标从 1 开始且长度为 n 的数组 result ，其中 result[k] 表示所有满足要求的房屋对的数量，即从一个房屋到另一个房屋需要经过的 最少 街道数为 k 。\n注意，x 与 y 可以 相等。你的代码需要通过以下测试:\n\n assert countOfPairs(n = 3, x = 1, y = 3) == [6,0,0]\n assert countOfPairs(n = 5, x = 2, y = 4) == [10,8,2,0,0] \n assert countOfPairs(n = 4, x = 1, y = 1) == [6,4,2,0] \n'),
+                dict(role='BOT', prompt="[BEGIN]\n 'from itertools import accumulate\ndef countOfPairs(n, x, y):\n        x, y = min(x, y), max(x, y)\n        A = [0] * n\n        for i in range(1, n + 1):\n            A[0] += 2                                   \n            A[min(i - 1, abs(i - y) + x)] -= 1          \n            A[min(n - i, abs(i - x) + 1 + n - y)] -= 1  \n            A[min(abs(i - x), abs(y - i) + 1)] += 1     \n            A[min(abs(i - x) + 1, abs(y - i))] += 1     \n            r = max(x - i, 0) + max(i - y, 0)\n            A[r + (y - x + 0) // 2] -= 1                \n            A[r + (y - x + 1) // 2] -= 1                \n        return list(accumulate(A))' \n[DONE] \n\n "),
+                dict(role='HUMAN', prompt='你是一名专业的 Python 程序员，下面是你的任务: 给你一个字符串 word，由 不同 小写英文字母组成。\n电话键盘上的按键与 不同 小写英文字母集合相映射，可以通过按压按键来组成单词。例如，按键 2 对应 ["a","b","c"]，我们需要按一次键来输入 "a"，按两次键来输入 "b"，按三次键来输入 "c"。\n现在允许你将编号为 2 到 9 的按键重新映射到 不同 字母集合。每个按键可以映射到 任意数量 的字母，但每个字母 必须 恰好 映射到 一个 按键上。你需要找到输入字符串 word 所需的 最少 按键次数。\n返回重新映射按键后输入 word 所需的 最少 按键次数。\n下面给出了一种电话键盘上字母到按键的映射作为示例。注意 1，*，# 和 0 不 对应任何字母。你的代码需要通过以下测试:\n\n assert minimumPushes("abcde") == 5 \n assert minimumPushes("xyzxyzxyzxyz") == 12 \n assert minimumPushes("aabbccddeeffgghhiiiiii") == 24 \n'),
+                dict(role='BOT', prompt="[BEGIN]\n 'def minimumPushes(word):\n        letter_counts = {}\n        for c in word:\n            letter_counts[c] = letter_counts.get(c, 0) + 1\n        counts = list(letter_counts.values())\n        counts.sort(reverse=True)\n        ans, row = 0, 1\n        for i in range(len(counts)):\n            if i > 7 and i % 8 == 0:\n                row += 1\n            ans += row * counts[i]\n        return ans' \n[DONE] \n\n "),
+                dict(role='HUMAN', prompt='你是一名专业的 Python 程序员，下面是你的任务: 给你一个整数 k 和一个整数 x 。\n令 s 为整数 num 的下标从 1 开始的二进制表示。我们说一个整数 num 的 价值 是满足 i % x == 0 且 s[i] 是 设置位 的 i 的数目。\n请你返回 最大 整数 num ，满足从 1 到 num 的所有整数的 价值 和小于等于 k 。\n注意：\n一个整数二进制表示下 设置位 是值为 1 的数位。\n一个整数的二进制表示下标从右到左编号，比方说如果 s == 11100 ，那么 s[4] == 1 且 s[2] == 0。你的代码需要通过以下测试:\n\n assert findMaximumNumber(k = 9, x = 1) == 6 \n assert findMaximumNumber(k = 7, x = 2) == 9 \n'),
+                dict(role='BOT', prompt="[BEGIN]\n 'def findMaximumNumber(k, x):\n        def check(v):\n            A = list(map(int, bin(v)[2:]))\n            n = len(A)\n            res = p = 0\n            for i,v in enumerate(A):\n                if v == 1:\n                    l = n - i - 1\n                    res += (p << l) + ((l // x) << (l - 1) if l else 0)\n                if (n - i) % x == 0:\n                    p += v\n            return res + p\n        l, r = 1, 10 ** 15\n        while l < r:\n            mid = (l + r + 1) // 2\n            if check(mid) <= k:\n                l = mid\n            else:\n                r = mid - 1\n        return l' \n[DONE] \n\n "),
+                dict(role='HUMAN', prompt='你是一名专业的 Python 程序员，下面是你的任务: {text} 你的代码需要通过以下测试:\n\n {test_list}  \n'),
+                dict(role='BOT', prompt='[BEGIN]\n'),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512),
+)
+
+
+LC_eval_cfg = dict(evaluator=dict(type=LCPassKEvaluator), pred_role='BOT')
+
+for difficulty in LC_difficulties_list:
+    compassbench_v1_3_code_datasets.append(
+        dict(
+            type=LCDataset,
+            abbr='compass_bench_code_interview_en-' + difficulty,
+            path='./data/compassbench_v1_3/coding/compass_bench_code_interview/compass_bench_code_interview_en.jsonl',
+            difficulty=difficulty,
+            reader_cfg=LC_reader_cfg,
+            infer_cfg=LC_en_infer_cfg,
+            eval_cfg=LC_eval_cfg,
+        )
+    )
+    compassbench_v1_3_code_datasets.append(
+        dict(
+            type=LCDataset,
+            abbr='compass_bench_code_interview_zh-' + difficulty,
+            path='./data/compassbench_v1_3/coding/compass_bench_code_interview/compass_bench_code_interview_zh.jsonl',
+            difficulty=difficulty,
+            reader_cfg=LC_reader_cfg,
+            infer_cfg=LC_cn_infer_cfg,
+            eval_cfg=LC_eval_cfg,
+        )
+    )
+
+
+# --------------------------------------------Code Competition(TACO) ---------------------------------------------------------------
+TACO_difficulties_list = ['EASY', 'MEDIUM', 'MEDIUM_HARD', 'HARD', 'VERY_HARD']
+TACO_reader_cfg = dict(input_columns=['question', 'starter'], output_column='problem_id', train_split='test')
+
+TACO_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='Please write a python program to address the following QUESTION. Your ANSWER should be in a code block format like this: ```python # Write your code here ```. \nQUESTION:\n{question} {starter}\nANSWER:\n'),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=1024),
+)
+
+TACO_eval_cfg = dict(evaluator=dict(type=TACOEvaluator), pred_role='BOT')
+
+for difficulty in TACO_difficulties_list:
+    compassbench_v1_3_code_datasets.append(
+        dict(
+            type=TACODataset,
+            abbr='TACO-' + difficulty,
+            path='./data/compassbench_v1_3/coding/compass_bench_code_competition',
+            difficulty=difficulty,
+            reader_cfg=TACO_reader_cfg,
+            infer_cfg=TACO_infer_cfg,
+            eval_cfg=TACO_eval_cfg,
+        )
+    )

opencompass/configs/datasets/compassbench_v1_3/compassbench_v1_3_knowledge.py

@@ -0,0 +1,94 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import CircularEvaluator, AccEvaluator
+from opencompass.datasets.compassbench_obj import (
+    CompassBenchObjectiveV1_3,
+    compassbench_objective_v1_3_postprocess,
+)
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+
+prompt_cn = {
+    'single_choice_cn': '以下是一道单项选择题，请你根据你了解的知识给出正确的答案选项。请你一步步推理并在最后用“答案选项为X”来回答，其中X是ABCD中你认为正确的选项序号\n下面是你要回答的题目：\n{question}\n让我们一步步解决这个问题：',
+    'cloze_cn': '以下是一道填空题，请你根据你了解的知识一步步思考后把你的最终答案放到\\boxed{}中。\n下面是你要回答的题目：\n{question}\n让我们一步步解决这个问题：',
+}
+
+prompt_en = {
+    'single_choice_en': "Here is a single-choice question. Please give the correct answer based on your knowledge. Please reason step by step and answer with 'The answer is X' at the end, where X is the option letter you think is correct.\nHere is the question you need to answer:\n{question}\nLet's solve this problem step by step:",
+    'cloze_en': "Here is a fill-in-the-blank question. Please think step by step based on your knowledge and put your final answer in \\boxed{}. Here is the question you need to answer:\n{question}\nLet's solve this problem step by step:",
+}
+
+douknow_sets = {
+    'wiki_en_sub_500_人文科学':['single_choice_en'],
+    'wiki_en_sub_500_社会科学':['single_choice_en'],
+    'wiki_en_sub_500_生活常识':['single_choice_en'],
+    'wiki_en_sub_500_自然科学-工科':['single_choice_en'],
+    'wiki_en_sub_500_自然科学-理科':['single_choice_en'],
+    'wiki_zh_sub_500_人文科学': ['single_choice_cn'],
+    'wiki_zh_sub_500_社会科学': ['single_choice_cn'],
+    'wiki_zh_sub_500_生活常识': ['single_choice_cn'],
+    'wiki_zh_sub_500_自然科学-工科':['single_choice_cn'],
+    'wiki_zh_sub_500_自然科学-理科':['single_choice_cn'],
+}
+
+data_path = './data/compassbench_v1_3/knowledge'
+
+# Set up the prompts
+CircularEval = True
+
+compassbench_knowledge_datasets = []
+
+for _split in list(douknow_sets.keys()):
+    for _name in douknow_sets[_split]:
+        if 'cn' in _name:
+            single_choice_prompts = prompt_cn
+            cloze_prompts = prompt_cn
+        else:
+            single_choice_prompts = prompt_en
+            cloze_prompts = prompt_en
+
+        if 'single_choice' in _name:
+            template_round = [dict(role='HUMAN', prompt=single_choice_prompts[_name])]
+            pred_postprocessor = dict(type=first_option_postprocess, options='ABCD')
+            evaluator = dict(type=CircularEvaluator if CircularEval else AccEvaluator)
+            dataset_name = _name + '_circular' if CircularEval else _name
+            dataset_abbr = (
+                'compassbench-' + _split + '_circular'
+                if CircularEval
+                else 'compassbench-' + _split
+            )
+        else:
+            template_round = [dict(role='HUMAN', prompt=cloze_prompts[_name])]
+            pred_postprocessor = dict(
+                type=compassbench_objective_v1_3_postprocess, name=_name
+            )
+            evaluator = dict(type=AccEvaluator)
+            dataset_name = _name
+            dataset_abbr = 'compassbench-' + _split
+
+        douknow_infer_cfg = dict(
+            prompt_template=dict(
+                type=PromptTemplate, template=dict(round=template_round)
+            ),
+            retriever=dict(type=ZeroRetriever),
+            inferencer=dict(type=GenInferencer, max_out_len=2048),
+        )
+
+        douknow_eval_cfg = dict(
+            evaluator=evaluator,
+            pred_postprocessor=pred_postprocessor,
+        )
+
+        compassbench_knowledge_datasets.append(
+            dict(
+                type=CompassBenchObjectiveV1_3,
+                path=f'{data_path}/{_split}.jsonl',
+                name=dataset_name,
+                abbr=dataset_abbr,
+                reader_cfg=dict(input_columns=['question'], output_column='answer'),
+                infer_cfg=douknow_infer_cfg,
+                eval_cfg=douknow_eval_cfg,
+            )
+        )
+del _split, _name

opencompass/configs/datasets/compassbench_v1_3/compassbench_v1_3_math.py

@@ -0,0 +1,86 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import CircularEvaluator, AccEvaluator
+from opencompass.datasets.compassbench_obj import CompassBenchObjectiveV1_3
+from opencompass.datasets import MATHEvaluator, math_postprocess_v2
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+
+prompt_cn = {
+    'single_choice_cn': '以下是一道单项选择题，请你根据你了解的知识给出正确的答案选项。请你一步步推理并在最后用“答案选项为X”来回答，其中X是ABCD中你认为正确的选项序号\n下面是你要回答的题目：\n{question}\n让我们一步步解决这个问题：',
+    'cloze_cn': '以下是一道数学计算题，请你一步一步计算，并在最后用\\boxed{}包裹并返回你计算的最终答案。\n下面是你要回答的题目：\n{question}\n让我们一步步解决这个问题：',
+}
+
+prompt_en = {
+    'single_choice_en': "Here is a single-choice question. Please give the correct answer based on your knowledge. Please reason step by step and answer with 'The answer is X' at the end, where X is the option number you think is correct.\nHere is the question you need to answer:\n{question}\nLet's solve this problem step by step:",
+    'cloze_en': 'Here is a arithematic problem. Please reason step by step, and put your final answer within \\boxed{}. Here is the question you need to answer:\n{question}\nLet\'s solve this problem step by step:',
+}
+
+
+douknow_sets = {
+    'arithmetic_cloze_en': ['cloze_en'],
+    'college_single_choice_en': ['single_choice_en'],
+    'college_single_choice_cn': ['single_choice_cn'],
+}
+
+data_path = './data/compassbench_v1_3/math'
+
+# Set up the prompts
+CircularEval = True
+
+compassbench_math_datasets = []
+
+for _split in list(douknow_sets.keys()):
+    for _name in douknow_sets[_split]:
+        if 'cn' in _name:
+            single_choice_prompts = prompt_cn
+            cloze_prompts = prompt_cn
+        else:
+            single_choice_prompts = prompt_en
+            cloze_prompts = prompt_en
+
+        if 'single_choice' in _name:
+            template_round = [dict(role='HUMAN', prompt=single_choice_prompts[_name])]
+            pred_postprocessor = dict(type=first_option_postprocess, options='ABCD')
+            evaluator = dict(type=CircularEvaluator if CircularEval else AccEvaluator)
+            dataset_name = _name + '_circular' if CircularEval else _name
+            dataset_abbr = (
+                'compassbench-' + _split + '_circular'
+                if CircularEval
+                else 'compassbench-' + _split
+            )
+        else:
+            template_round = [dict(role='HUMAN', prompt=cloze_prompts[_name])]
+            pred_postprocessor = dict(
+                type=math_postprocess_v2,
+            )
+            evaluator = dict(type=MATHEvaluator)
+            dataset_name = _name
+            dataset_abbr = 'compassbench-' + _split
+
+        douknow_infer_cfg = dict(
+            prompt_template=dict(
+                type=PromptTemplate, template=dict(round=template_round)
+            ),
+            retriever=dict(type=ZeroRetriever),
+            inferencer=dict(type=GenInferencer, max_out_len=2048),
+        )
+
+        douknow_eval_cfg = dict(
+            evaluator=evaluator,
+            pred_postprocessor=pred_postprocessor,
+        )
+
+        compassbench_math_datasets.append(
+            dict(
+                type=CompassBenchObjectiveV1_3,
+                path=f'{data_path}/{_split}.jsonl',
+                name=dataset_name,
+                abbr=dataset_abbr,
+                reader_cfg=dict(input_columns=['question'], output_column='answer'),
+                infer_cfg=douknow_infer_cfg,
+                eval_cfg=douknow_eval_cfg,
+            )
+        )
+del _split, _name

opencompass/configs/datasets/compassbench_v1_3/compassbench_v1_3_objective_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .compassbench_v1_3_objective_gen_068af0 import compassbench_aug_datasets  # noqa: F401, F403

opencompass/configs/datasets/compassbench_v1_3/compassbench_v1_3_objective_gen_068af0.py

@@ -0,0 +1,74 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import CircularEvaluator, AccEvaluator
+from opencompass.datasets.compassbench_obj import CompassBenchObjectiveV1_3, compassbench_objective_v1_3_postprocess
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+
+prompt_cn = {
+    'single_choice_cn': '以下是一道单项选择题，请你根据你了解的知识给出正确的答案选项。请你一步步推理并在最后用“答案选项为X”来回答，其中X是ABCD中你认为正确的选项序号\n下面是你要回答的题目：\n{question}\n让我们一步步解决这个问题：',
+    'cloze_cn': '以下是一道填空题，请你根据你了解的知识一步步思考后把你的最终答案放到\\boxed{}中。\n下面是你要回答的题目：\n{question}\n让我们一步步解决这个问题：',
+}
+
+prompt_en = {
+    'single_choice_en': "Here is a single-choice question. Please give the correct answer based on your knowledge. Please reason step by step and answer with 'The answer is X' at the end, where X is the option number you think is correct.\nHere is the question you need to answer:\n{question}\nLet's solve this problem step by step:",
+    'cloze_en': "Here is a fill-in-the-blank question. Please think step by step based on your knowledge and put your final answer in \\boxed{}. Here is the question you need to answer:\n{question}\nLet's solve this problem step by step:",
+}
+
+
+douknow_sets = {
+    'knowledge': ['single_choice_cn'],
+    'math': ['single_choice_cn'],
+}
+
+# Set up the prompts
+CircularEval = True
+
+
+compassbench_aug_datasets = []
+
+for _split in list(douknow_sets.keys()):
+    for _name in douknow_sets[_split]:
+        if 'cn' in _name:
+            single_choice_prompts = prompt_cn
+            cloze_prompts = prompt_cn
+        else:
+            single_choice_prompts = prompt_en
+            cloze_prompts = prompt_en
+        douknow_infer_cfg = dict(
+            ice_template=dict(
+                type=PromptTemplate,
+                template=dict(
+                    begin='</E>',
+                    round=[
+                        dict(
+                            role='HUMAN',
+                            prompt= single_choice_prompts[_name],
+                        ),
+                        dict(role='BOT', prompt='{answer}'),] if 'choice' in _name else cloze_prompts[_name],
+                    ),
+                ice_token='</E>',
+            ),
+            retriever=dict(type=ZeroRetriever),
+            inferencer=dict(type=GenInferencer),
+        )
+        douknow_eval_cfg = dict(
+            evaluator=dict(type=CircularEvaluator if CircularEval else AccEvaluator) if 'single_choice' in _name else dict(type=AccEvaluator),
+            pred_postprocessor=dict(type=first_option_postprocess, options='ABCD' ) if 'single_choice' in _name else dict(type=compassbench_objective_v1_3_postprocess, name=_name))
+
+        compassbench_aug_datasets.append(
+            dict(
+                type=CompassBenchObjectiveV1_3,
+                path=f'./data/compassbench_v1_3/{_split}/{_name}.jsonl',
+                name='circular_' + _name if CircularEval else _name,
+                abbr='compassbench-' + _split + '-' + _name + 'circular'if CircularEval else '',
+                reader_cfg=dict(
+                    input_columns=['question'],
+                    output_column='answer'
+                    ),
+                infer_cfg=douknow_infer_cfg,
+                eval_cfg=douknow_eval_cfg,
+            ))
+
+del _split, _name

opencompass/configs/datasets/compassbench_v1_3/compassbench_v1_3_prompt.py

@@ -0,0 +1,44 @@
+FORCE_STOP_PROMPT_EN = (
+    """You should directly give results based on history information."""
+)
+
+FEWSHOT_INSTRUCTION = """\
+You are an assistant who can utilize external tools.
+{tool_description}
+To use a tool, please response with the following format:
+```
+{thought} Think what you need to solve, do you need to use tools?
+{action} The tool name, should be one of [{action_names}].
+{action_input} The input to the tool that you want to use.
+```
+The tool will give you response after your response using the following format:
+```
+{response} the results after call the tool.
+```
+Therefore DO NOT generate tool response by yourself.
+
+Also please follow the guidelines:
+1. Always use code interpreter to solve the problem.
+2. The generated codes should always in a markdown code block format.
+3. The generated codes will be executed in an ipython manner and the results will be cached.
+4. Your responded code should always be simple and only solves the problem in current step.
+
+For example:
+
+File url: `xxxx`
+### Step 1. Load the dataset from the url into a pandas DataFrame named `df`.
+
+{thought} We should use `pandas` to solve this step.
+{action} IPythonInterpreter
+{action_input} ```python
+import pandas as pd
+url = "xxxx"
+data = pd.read_csv(url)
+```
+{response} The code is succeed without any outputs.
+
+Let us begin from here!
+"""
+
+IPYTHON_INTERPRETER_DESCRIPTION = """\
+It can run Python code in a manner as jupyter notebook. The code must be a valid code that contains only python method."""

opencompass/configs/datasets/hungarian_exam/hungarian_exam_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .hungarian_exam_gen_8a1435 import hungarianmath_datasets  # noqa: F401, F403

opencompass/configs/datasets/hungarian_exam/hungarian_exam_gen_8a1435.py

@@ -0,0 +1,91 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HungarianExamMathDataset
+
+hungarianmath_reader_cfg = dict(input_columns=['question'], output_column=None)
+
+template = """Problem:
+Find the domain of the expression $\frac{\sqrt{x-2}}{\sqrt{5-x}}$.
+
+Solution:
+To determine the domain, we must ensure that:
+1. The expressions inside each square root are non-negative.
+2. The denominator is not equal to zero.
+
+For the numerator, $x-2 \ge 0$ gives $x \ge 2$.
+
+For the denominator, $5-x \ge 0$ gives $x \le 5$. And since the denominator cannot be zero, $5-x > 0$ which further narrows it to $x < 5$.
+
+Combining these results, the domain of the expression is $[2,5)$.
+
+Final Answer: The final answer is $[2,5)$.
+
+Problem:
+If $\det \mathbf{A} = 2$ and $\det \mathbf{B} = 12$, then find $\det (\mathbf{A} \mathbf{B})$.
+
+Solution:
+Using the property of determinants, we can say that:
+$\det (\mathbf{A} \mathbf{B}) = (\det \mathbf{A})(\det \mathbf{B})$.
+Plugging in the given values:
+$\det (\mathbf{A} \mathbf{B}) = 2 \times 12 = 24$.
+
+Final Answer: The final answer is $24$.
+
+Problem:
+Terrell usually lifts two 20-pound weights 12 times. If he uses two 15-pound weights instead, how many times must Terrell lift them in order to lift the same total weight?
+
+Solution:
+First, calculate the total weight Terrell lifts with the 20-pound weights:
+$2 \times 12 \times 20 = 480$ pounds.
+If he uses 15-pound weights and lifts them $n$ times:
+$2 \times 15 \times n = 30n$ pounds.
+To find $n$, set these two equal:
+\begin{align*}
+30n &= 480 \\
+n &= \frac{480}{30} \\
+n &= 16
+\end{align*}
+
+Final Answer: The final answer is $16$.
+
+Problem:
+If the system of equations
+\begin{align*}
+6x-4y &= a, \\
+6y-9x &= b.
+\end{align*}
+has a solution $(x, y)$ where $x$ and $y$ are both nonzero, find $\frac{a}{b}$, assuming $b$ is nonzero.
+
+Solution:
+Multiply the first equation by $-\frac{3}{2}$ to obtain:
+$6y-9x = -\frac{3}{2}a$.
+Since we also know that $6y-9x = b$, equating them gives:
+$-\frac{3}{2}a = b$ which implies $\frac{a}{b} = -\frac{2}{3}$.
+
+Final Answer: The final answer is $-\frac{2}{3}$."""
+
+hungarianmath_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt=template+'\n\nProblem:\n{question}\n\nSolution:\n'),
+            ],
+        )),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=1024))
+
+# Attention: this math dataset needs human to evaluate the generated answer, so the AccEvaluator is just a placeholder.
+hungarianmath_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+hungarianmath_datasets = [
+    dict(
+        abbr='HungarianExamMath',
+        type=HungarianExamMathDataset,
+        path='./data/HungarianExamMath/test.csv',
+        reader_cfg=hungarianmath_reader_cfg,
+        infer_cfg=hungarianmath_infer_cfg,
+        eval_cfg=hungarianmath_eval_cfg)
+]

opencompass/configs/datasets/lambada/lambada_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .lambada_gen_217e11 import lambada_datasets  # noqa: F401, F403

opencompass/configs/datasets/lambada/lambada_gen_217e11.py

@@ -0,0 +1,33 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import lambadaDataset, LambadaEvaluator
+
+lambada_reader_cfg = dict(
+    input_columns=['prompt'],
+    output_column='label',
+    train_split='test',
+    test_split='test')
+
+lambada_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='Please complete the following sentence:\n{prompt}')
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=5))
+
+lambada_eval_cfg = dict(evaluator=dict(type=LambadaEvaluator))
+
+lambada_datasets = [
+    dict(
+        abbr='lambada',
+        type=lambadaDataset,
+        path='opencompass/lambada',
+        reader_cfg=lambada_reader_cfg,
+        infer_cfg=lambada_infer_cfg,
+        eval_cfg=lambada_eval_cfg)
+]

opencompass/configs/datasets/lambada/lambada_gen_8b48a5.py

@@ -0,0 +1,29 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import lambadaDataset, LambadaEvaluator
+
+lambada_reader_cfg = dict(
+    input_columns=['prompt'],
+    output_column='label',
+    train_split='test',
+    test_split='test')
+
+lambada_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template='Please complete the following sentence: {prompt}'),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=5))
+
+lambada_eval_cfg = dict(evaluator=dict(type=LambadaEvaluator))
+
+lambada_datasets = [
+    dict(
+        abbr='lambada',
+        type=lambadaDataset,
+        path='opencompass/lambada',
+        reader_cfg=lambada_reader_cfg,
+        infer_cfg=lambada_infer_cfg,
+        eval_cfg=lambada_eval_cfg)
+]

opencompass/configs/datasets/subjective/alignbench/alignbench_judgeby_critiquellm.py

@@ -0,0 +1,67 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import LMEvaluator
+from opencompass.summarizers import AlignmentBenchSummarizer
+from opencompass.datasets import AlignmentBenchDataset
+
+subjective_reader_cfg = dict(
+    input_columns=['question', 'capability', 'critiquellm_prefix'],
+    output_column='judge',
+    )
+
+subjective_all_sets = [
+    'alignment_bench',
+]
+data_path ='data/subjective/alignment_bench'
+
+alignment_bench_config_path = 'data/subjective/alignment_bench/config'
+alignment_bench_config_name = 'multi-dimension'
+
+alignbench_datasets = []
+
+for _name in subjective_all_sets:
+    subjective_infer_cfg = dict(
+            prompt_template=dict(
+                type=PromptTemplate,
+                template=dict(round=[
+                    dict(
+                        role='HUMAN',
+                        prompt='{question}'
+                    ),
+                ]),
+            ),
+            retriever=dict(type=ZeroRetriever),
+            inferencer=dict(type=GenInferencer, max_out_len=2048),
+        )
+
+    subjective_eval_cfg = dict(
+        evaluator=dict(
+            type=LMEvaluator,
+            prompt_template=dict(
+                type=PromptTemplate,
+                template=dict(round=[
+                    dict(
+                        role='HUMAN',
+                        prompt = '{critiquellm_prefix}[助手的答案开始]\n{prediction}\n[助手的答案结束]\n'
+                    ),
+                ]),
+            ),
+        ),
+        pred_role='BOT',
+    )
+
+    alignbench_datasets.append(
+        dict(
+            abbr=f'{_name}',
+            type=AlignmentBenchDataset,
+            path=data_path,
+            name=_name,
+            alignment_bench_config_path=alignment_bench_config_path,
+            alignment_bench_config_name=alignment_bench_config_name,
+            reader_cfg=subjective_reader_cfg,
+            infer_cfg=subjective_infer_cfg,
+            eval_cfg=subjective_eval_cfg,
+            mode='singlescore',
+            summarizer = dict(type=AlignmentBenchSummarizer, judge_type='general')
+        ))

opencompass/configs/datasets/subjective/alignbench/alignbench_judgeby_critiquellm_new.py

@@ -0,0 +1,66 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import LMEvaluator
+from opencompass.datasets import AlignmentBenchDataset, alignbench_postprocess
+
+subjective_reader_cfg = dict(
+    input_columns=['question', 'capability', 'critiquellm_prefix'],
+    output_column='judge',
+    )
+
+subjective_all_sets = [
+    'alignment_bench',
+]
+data_path ='data/subjective/alignment_bench'
+
+alignment_bench_config_path = 'data/subjective/alignment_bench/config'
+alignment_bench_config_name = 'multi-dimension'
+
+alignbench_datasets = []
+
+for _name in subjective_all_sets:
+    subjective_infer_cfg = dict(
+            prompt_template=dict(
+                type=PromptTemplate,
+                template=dict(round=[
+                    dict(
+                        role='HUMAN',
+                        prompt='{question}'
+                    ),
+                ]),
+            ),
+            retriever=dict(type=ZeroRetriever),
+            inferencer=dict(type=GenInferencer, max_out_len=2048),
+        )
+
+    subjective_eval_cfg = dict(
+        evaluator=dict(
+            type=LMEvaluator,
+            prompt_template=dict(
+                type=PromptTemplate,
+                template=dict(round=[
+                    dict(
+                        role='HUMAN',
+                        prompt = '{critiquellm_prefix}[助手的答案开始]\n{prediction}\n[助手的答案结束]\n'
+                    ),
+                ]),
+            ),
+            dict_postprocessor=dict(type=alignbench_postprocess, judge_type='general'),
+        ),
+        pred_role='BOT',
+    )
+
+    alignbench_datasets.append(
+        dict(
+            abbr=f'{_name}',
+            type=AlignmentBenchDataset,
+            path=data_path,
+            name=_name,
+            alignment_bench_config_path=alignment_bench_config_path,
+            alignment_bench_config_name=alignment_bench_config_name,
+            reader_cfg=subjective_reader_cfg,
+            infer_cfg=subjective_infer_cfg,
+            eval_cfg=subjective_eval_cfg,
+            mode='singlescore',
+        ))

opencompass/configs/datasets/subjective/alignbench/alignbench_v1_1_judgeby_critiquellm.py

@@ -0,0 +1,67 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import LMEvaluator
+from opencompass.summarizers import AlignmentBenchSummarizer
+from opencompass.datasets import AlignmentBenchDataset
+
+subjective_reader_cfg = dict(
+    input_columns=['question', 'capability', 'critiquellm_prefix'],
+    output_column='judge',
+    )
+
+subjective_all_sets = [
+    'alignment_bench_v1_1', # Changed to Alignbench_v1_1 since 06/15/2024, refer to https://github.com/THUDM/AlignBench
+]
+data_path ='data/subjective/alignment_bench'
+
+alignment_bench_config_path = 'data/subjective/alignment_bench/config'
+alignment_bench_config_name = 'multi-dimension'
+
+alignbench_datasets = []
+
+for _name in subjective_all_sets:
+    subjective_infer_cfg = dict(
+            prompt_template=dict(
+                type=PromptTemplate,
+                template=dict(round=[
+                    dict(
+                        role='HUMAN',
+                        prompt='{question}'
+                    ),
+                ]),
+            ),
+            retriever=dict(type=ZeroRetriever),
+            inferencer=dict(type=GenInferencer, max_out_len=2048),
+        )
+
+    subjective_eval_cfg = dict(
+        evaluator=dict(
+            type=LMEvaluator,
+            prompt_template=dict(
+                type=PromptTemplate,
+                template=dict(round=[
+                    dict(
+                        role='HUMAN',
+                        prompt = '{critiquellm_prefix}[助手的答案开始]\n{prediction}\n[助手的答案结束]\n'
+                    ),
+                ]),
+            ),
+        ),
+        pred_role='BOT',
+    )
+
+    alignbench_datasets.append(
+        dict(
+            abbr=f'{_name}',
+            type=AlignmentBenchDataset,
+            path=data_path,
+            name=_name,
+            alignment_bench_config_path=alignment_bench_config_path,
+            alignment_bench_config_name=alignment_bench_config_name,
+            reader_cfg=subjective_reader_cfg,
+            infer_cfg=subjective_infer_cfg,
+            eval_cfg=subjective_eval_cfg,
+            mode='singlescore',
+            summarizer = dict(type=AlignmentBenchSummarizer, judge_type='general')
+        ))

opencompass/configs/datasets/subjective/alignbench/alignbench_v1_1_judgeby_critiquellm_new.py

@@ -0,0 +1,66 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import LMEvaluator
+from opencompass.datasets import AlignmentBenchDataset, alignbench_postprocess
+
+subjective_reader_cfg = dict(
+    input_columns=['question', 'capability', 'critiquellm_prefix'],
+    output_column='judge',
+    )
+
+subjective_all_sets = [
+    'alignment_bench_v1_1', # Changed to Alignbench_v1_1 since 06/15/2024, refer to https://github.com/THUDM/AlignBench
+]
+data_path ='data/subjective/alignment_bench'
+
+alignment_bench_config_path = 'data/subjective/alignment_bench/config'
+alignment_bench_config_name = 'multi-dimension'
+
+alignbench_datasets = []
+
+for _name in subjective_all_sets:
+    subjective_infer_cfg = dict(
+            prompt_template=dict(
+                type=PromptTemplate,
+                template=dict(round=[
+                    dict(
+                        role='HUMAN',
+                        prompt='{question}'
+                    ),
+                ]),
+            ),
+            retriever=dict(type=ZeroRetriever),
+            inferencer=dict(type=GenInferencer, max_out_len=2048),
+        )
+
+    subjective_eval_cfg = dict(
+        evaluator=dict(
+            type=LMEvaluator,
+            prompt_template=dict(
+                type=PromptTemplate,
+                template=dict(round=[
+                    dict(
+                        role='HUMAN',
+                        prompt = '{critiquellm_prefix}[助手的答案开始]\n{prediction}\n[助手的答案结束]\n'
+                    ),
+                ]),
+            ),
+            dict_postprocessor=dict(type=alignbench_postprocess, judge_type='general'),
+        ),
+        pred_role='BOT',
+    )
+
+    alignbench_datasets.append(
+        dict(
+            abbr=f'{_name}',
+            type=AlignmentBenchDataset,
+            path=data_path,
+            name=_name,
+            alignment_bench_config_path=alignment_bench_config_path,
+            alignment_bench_config_name=alignment_bench_config_name,
+            reader_cfg=subjective_reader_cfg,
+            infer_cfg=subjective_infer_cfg,
+            eval_cfg=subjective_eval_cfg,
+            mode='singlescore',
+        ))