Add files using upload-large-folder tool

opencompass/configs/datasets/CLUE_DRCD/CLUE_DRCD_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .CLUE_DRCD_gen_1bd3c8 import DRCD_datasets  # noqa: F401, F403

opencompass/configs/datasets/CLUE_DRCD/CLUE_DRCD_gen_1bd3c8.py

@@ -0,0 +1,36 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import EMEvaluator
+from opencompass.datasets import DRCDDataset, drcd_postprocess
+
+DRCD_reader_cfg = dict(
+    input_columns=['question', 'context'], output_column='answers')
+
+DRCD_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='根据文章回答问题。你的答案应该尽可能简练，请以 ‘答案是’ 开头的句式作答。\n文章：{context}\n问：{question}\n答：'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+DRCD_eval_cfg = dict(
+    evaluator=dict(type=EMEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=drcd_postprocess),
+
+)
+
+DRCD_datasets = [
+    dict(
+        type=DRCDDataset,
+        abbr='DRCD_dev',
+        path='opencompass/drcd_dev',
+        reader_cfg=DRCD_reader_cfg,
+        infer_cfg=DRCD_infer_cfg,
+        eval_cfg=DRCD_eval_cfg),
+]

opencompass/configs/datasets/CLUE_DRCD/CLUE_DRCD_gen_3749cd.py

@@ -0,0 +1,33 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import EMEvaluator
+from opencompass.datasets import DRCDDataset
+
+DRCD_reader_cfg = dict(
+    input_columns=['question', 'context'], output_column='answers')
+
+DRCD_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(role='HUMAN', prompt='文章：{context}\n根据上文，回答如下问题：{question}'),
+            dict(role='BOT', prompt='答：'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+DRCD_eval_cfg = dict(
+    evaluator=dict(type=EMEvaluator),
+    pred_role='BOT',
+)
+
+DRCD_datasets = [
+    dict(
+        type=DRCDDataset,
+        abbr='DRCD_dev',
+        path='opencompass/drcd_dev',
+        reader_cfg=DRCD_reader_cfg,
+        infer_cfg=DRCD_infer_cfg,
+        eval_cfg=DRCD_eval_cfg),
+]

opencompass/configs/datasets/CLUE_DRCD/CLUE_DRCD_gen_8484b9.py

@@ -0,0 +1,27 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import EMEvaluator
+from opencompass.datasets import DRCDDataset
+
+DRCD_reader_cfg = dict(
+    input_columns=['question', 'context'], output_column='answers')
+
+DRCD_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template='文章：{context}\n根据上文，回答如下问题： {question}\n答：'),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+DRCD_eval_cfg = dict(evaluator=dict(type=EMEvaluator), )
+
+DRCD_datasets = [
+    dict(
+        type=DRCDDataset,
+        abbr='DRCD_dev',
+        path='opencompass/drcd_dev',
+        reader_cfg=DRCD_reader_cfg,
+        infer_cfg=DRCD_infer_cfg,
+        eval_cfg=DRCD_eval_cfg),
+]

opencompass/configs/datasets/CLUE_DRCD/CLUE_DRCD_gen_941108.py

@@ -0,0 +1,34 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import EMEvaluator
+from opencompass.datasets import DRCDDataset
+
+DRCD_reader_cfg = dict(
+    input_columns=['question', 'context'], output_column='answers')
+
+DRCD_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='文章：{context}\n根据上文，回答如下问题：\n{question}\n答：'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+DRCD_eval_cfg = dict(
+    evaluator=dict(type=EMEvaluator),
+    pred_role='BOT',
+)
+
+DRCD_datasets = [
+    dict(
+        type=DRCDDataset,
+        abbr='DRCD_dev',
+        path='opencompass/drcd_dev',
+        reader_cfg=DRCD_reader_cfg,
+        infer_cfg=DRCD_infer_cfg,
+        eval_cfg=DRCD_eval_cfg),
+]

opencompass/configs/datasets/CLUE_ocnli/CLUE_ocnli_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .CLUE_ocnli_gen_c4cb6c import ocnli_datasets  # noqa: F401, F403

opencompass/configs/datasets/CLUE_ocnli/CLUE_ocnli_gen_51e956.py

@@ -0,0 +1,44 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CMNLIDatasetV2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+ocnli_reader_cfg = dict(
+    input_columns=['sentence1', 'sentence2'],
+    output_column='label',
+)
+
+# TODO: two prompt templates for ocnli
+ocnli_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                '阅读文章：{sentence1}\n根据上文，回答如下问题：{sentence2}\nA. 对\nB. 错\nC. 可能\n请从“A”，“B”，“C”中进行选择。\n答：'
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+ocnli_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+ocnli_datasets = [
+    dict(
+        abbr='ocnli',
+        type=CMNLIDatasetV2,  # ocnli share the same format with cmnli
+        path='opencompass/OCNLI-dev',
+        reader_cfg=ocnli_reader_cfg,
+        infer_cfg=ocnli_infer_cfg,
+        eval_cfg=ocnli_eval_cfg,
+    )
+]

opencompass/configs/datasets/CLUE_ocnli/CLUE_ocnli_gen_c4cb6c.py

@@ -0,0 +1,44 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CMNLIDatasetV2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+ocnli_reader_cfg = dict(
+    input_columns=['sentence1', 'sentence2'],
+    output_column='label',
+)
+
+# TODO: two prompt templates for ocnli
+ocnli_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                '语句一：“{sentence1}”\n语句二：“{sentence2}”\n请问这两句话是什么关系？\nA. 蕴含\n B. 矛盾\n C. 无关\n请从“A”，“B”，“C”中进行选择。\n答：'
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+ocnli_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+ocnli_datasets = [
+    dict(
+        abbr='ocnli',
+        type=CMNLIDatasetV2,  # ocnli share the same format with cmnli
+        path='opencompass/OCNLI-dev',
+        reader_cfg=ocnli_reader_cfg,
+        infer_cfg=ocnli_infer_cfg,
+        eval_cfg=ocnli_eval_cfg,
+    )
+]

opencompass/configs/datasets/CLUE_ocnli/CLUE_ocnli_ppl.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .CLUE_ocnli_ppl_fdc6de import ocnli_datasets  # noqa: F401, F403

opencompass/configs/datasets/CLUE_ocnli/CLUE_ocnli_ppl_98dd6e.py

@@ -0,0 +1,35 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HFDataset
+
+ocnli_reader_cfg = dict(
+    input_columns=['sentence1', 'sentence2'], output_column='label')
+
+# TODO: two prompt templates for ocnli
+ocnli_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'contradiction':
+            '阅读文章：{sentence1}\n根据上文，回答如下问题： {sentence2}？\n答：错',
+            'entailment': '阅读文章：{sentence1}\n根据上文，回答如下问题： {sentence2}？\n答：对',
+            'neutral': '如果{sentence1}为真，那么{sentence2}也为真吗?可能'
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+ocnli_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
+
+ocnli_datasets = [
+    dict(
+        type=HFDataset,
+        abbr='ocnli',
+        path='json',
+        split='train',
+        data_files='./data/CLUE/OCNLI/dev.json',
+        reader_cfg=ocnli_reader_cfg,
+        infer_cfg=ocnli_infer_cfg,
+        eval_cfg=ocnli_eval_cfg)
+]

opencompass/configs/datasets/CLUE_ocnli/CLUE_ocnli_ppl_ef69e7.py

@@ -0,0 +1,51 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HFDataset
+
+ocnli_reader_cfg = dict(
+    input_columns=['sentence1', 'sentence2'], output_column='label')
+
+# TODO: two prompt templates for ocnli
+ocnli_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'contradiction':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='阅读文章：{sentence1}\n根据上文，回答如下问题：{sentence2}？'),
+                dict(role='BOT', prompt='错')
+            ]),
+            'entailment':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='阅读文章：{sentence1}\n根据上文，回答如下问题：{sentence2}？'),
+                dict(role='BOT', prompt='对')
+            ]),
+            'neutral':
+            dict(round=[
+                dict(
+                    role='HUMAN', prompt='如果{sentence1}为真，那么{sentence2}也为真吗？'),
+                dict(role='BOT', prompt='可能')
+            ]),
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+ocnli_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
+
+ocnli_datasets = [
+    dict(
+        type=HFDataset,
+        abbr='ocnli',
+        path='json',
+        split='train',
+        data_files='./data/CLUE/OCNLI/dev.json',
+        reader_cfg=ocnli_reader_cfg,
+        infer_cfg=ocnli_infer_cfg,
+        eval_cfg=ocnli_eval_cfg)
+]

opencompass/configs/datasets/CLUE_ocnli/CLUE_ocnli_ppl_fdc6de.py

@@ -0,0 +1,55 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HFDataset
+
+ocnli_reader_cfg = dict(
+    input_columns=['sentence1', 'sentence2'], output_column='label')
+
+# TODO: two prompt templates for ocnli
+ocnli_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'contradiction':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='语句一：“{sentence1}”\n语句二：“{sentence2}”\n请问这两句话是什么关系？'
+                ),
+                dict(role='BOT', prompt='矛盾')
+            ]),
+            'entailment':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='语句一：“{sentence1}”\n语句二：“{sentence2}”\n请问这两句话是什么关系？'
+                ),
+                dict(role='BOT', prompt='蕴含')
+            ]),
+            'neutral':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='语句一：“{sentence1}”\n语句二：“{sentence2}”\n请问这两句话是什么关系？'
+                ),
+                dict(role='BOT', prompt='无关')
+            ]),
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+ocnli_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
+
+ocnli_datasets = [
+    dict(
+        type=HFDataset,
+        abbr='ocnli',
+        path='json',
+        split='train',
+        data_files='./data/CLUE/OCNLI/dev.json',
+        reader_cfg=ocnli_reader_cfg,
+        infer_cfg=ocnli_infer_cfg,
+        eval_cfg=ocnli_eval_cfg)
+]

opencompass/configs/datasets/MedBench/medbench_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .medbench_gen_d44f24 import medbench_datasets  # noqa: F401, F403

opencompass/configs/datasets/MedBench/medbench_gen_0b4fff.py

@@ -0,0 +1,119 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import MedBenchDataset, MedBenchEvaluator, MedBenchEvaluator_Cloze, MedBenchEvaluator_CMeEE, MedBenchEvaluator_CMeIE, MedBenchEvaluator_CHIP_CDEE, MedBenchEvaluator_CHIP_CDN, MedBenchEvaluator_CHIP_CTC, MedBenchEvaluator_NLG, MedBenchEvaluator_TF, MedBenchEvaluator_DBMHG, MedBenchEvaluator_SMDoc, MedBenchEvaluator_IMCS_V2_MRG
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+medbench_reader_cfg = dict(
+    input_columns=['problem_input'], output_column='label')
+
+medbench_multiple_choices_sets = ['Med-Exam', 'DDx-basic', 'DDx-advanced', 'MedSafety'] # 选择题，用acc判断
+
+medbench_qa_sets = ['MedHC', 'MedMC', 'MedDG', 'MedSpeQA', 'MedTreat', 'CMB-Clin'] # 开放式QA，有标答
+
+medbench_cloze_sets = ['MedHG'] # 限定域QA，有标答
+
+medbench_single_choice_sets = ['DrugCA'] # 正确与否判断，有标答
+
+medbench_ie_sets = ['DBMHG', 'CMeEE', 'CMeIE', 'CHIP-CDEE', 'CHIP-CDN', 'CHIP-CTC', 'SMDoc', 'IMCS-V2-MRG'] # 判断识别的实体是否一致，用F1评价
+
+medbench_datasets = []
+
+for name in medbench_single_choice_sets + medbench_multiple_choices_sets:
+    medbench_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=[dict(role='HUMAN', prompt='{problem_input}')])),
+        retriever=dict(type=ZeroRetriever
+                       ),  # retriver 不起作用，以输入参数为准 (zero-shot / few-shot)
+        inferencer=dict(type=GenInferencer))
+
+    medbench_eval_cfg = dict(
+        evaluator=dict(type=MedBenchEvaluator), pred_role='BOT')
+
+    medbench_datasets.append(
+        dict(
+            type=MedBenchDataset,
+            path='./data/MedBench/' + name,
+            name=name,
+            abbr='medbench-' + name,
+            setting_name='zero-shot',
+            reader_cfg=medbench_reader_cfg,
+            infer_cfg=medbench_infer_cfg.copy(),
+            eval_cfg=medbench_eval_cfg.copy()))
+
+for name in medbench_qa_sets:
+    medbench_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=[dict(role='HUMAN', prompt='{problem_input}')])),
+        retriever=dict(type=ZeroRetriever
+                       ),  # retriver 不起作用，以输入参数为准 (zero-shot / few-shot)
+        inferencer=dict(type=GenInferencer))
+
+    medbench_eval_cfg = dict(
+        evaluator=dict(type=MedBenchEvaluator_NLG), pred_role='BOT')
+
+    medbench_datasets.append(
+        dict(
+            type=MedBenchDataset,
+            path='./data/MedBench/' + name,
+            name=name,
+            abbr='medbench-' + name,
+            setting_name='zero-shot',
+            reader_cfg=medbench_reader_cfg,
+            infer_cfg=medbench_infer_cfg.copy(),
+            eval_cfg=medbench_eval_cfg.copy()))
+
+for name in medbench_cloze_sets:
+    medbench_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=[dict(role='HUMAN', prompt='{problem_input}')])),
+        retriever=dict(type=ZeroRetriever
+                       ),  # retriver 不起作用，以输入参数为准 (zero-shot / few-shot)
+        inferencer=dict(type=GenInferencer))
+
+    medbench_eval_cfg = dict(
+        evaluator=dict(type=MedBenchEvaluator_Cloze), pred_role='BOT')
+
+    medbench_datasets.append(
+        dict(
+            type=MedBenchDataset,
+            path='./data/MedBench/' + name,
+            name=name,
+            abbr='medbench-' + name,
+            setting_name='zero-shot',
+            reader_cfg=medbench_reader_cfg,
+            infer_cfg=medbench_infer_cfg.copy(),
+            eval_cfg=medbench_eval_cfg.copy()))
+
+for name in medbench_ie_sets:
+    medbench_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=[dict(role='HUMAN', prompt='{problem_input}')])),
+        retriever=dict(type=ZeroRetriever
+                       ),  # retriver 不起作用，以输入参数为准 (zero-shot / few-shot)
+        inferencer=dict(type=GenInferencer))
+
+    medbench_eval_cfg = dict(
+        evaluator=dict(type=eval('MedBenchEvaluator_'+name.replace('-', '_'))), pred_role='BOT')
+
+    medbench_datasets.append(
+        dict(
+            type=MedBenchDataset,
+            path='./data/MedBench/' + name,
+            name=name,
+            abbr='medbench-' + name,
+            setting_name='zero-shot',
+            reader_cfg=medbench_reader_cfg,
+            infer_cfg=medbench_infer_cfg.copy(),
+            eval_cfg=medbench_eval_cfg.copy()))
+
+del name, medbench_infer_cfg, medbench_eval_cfg

opencompass/configs/datasets/OpenFinData/OpenFinData_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .OpenFinData_gen_46dedb import OpenFinData_datasets  # noqa: F401, F403

opencompass/configs/datasets/OpenFinData/OpenFinData_gen_46dedb.py

@@ -0,0 +1,99 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets.OpenFinData import OpenFinDataDataset, OpenFinDataKWEvaluator
+from opencompass.utils.text_postprocessors import last_capital_postprocess
+
+OpenFinData_datasets = []
+OpenFinData_3choices_list = ['emotion_identification', 'entity_disambiguation', 'financial_facts']
+OpenFinData_4choices_list = ['data_inspection', 'financial_terminology', 'metric_calculation', 'value_extraction']
+OpenFinData_5choices_list = ['intent_understanding']
+OpenFinData_keyword_list = ['entity_recognition']
+OpenFinData_all_list = OpenFinData_3choices_list + OpenFinData_4choices_list + OpenFinData_5choices_list + OpenFinData_keyword_list
+
+OpenFinData_eval_cfg = dict(evaluator=dict(type=AccEvaluator), pred_postprocessor=dict(type=last_capital_postprocess))
+OpenFinData_KW_eval_cfg = dict(evaluator=dict(type=OpenFinDataKWEvaluator))
+
+for _name in OpenFinData_all_list:
+    if _name in OpenFinData_3choices_list:
+        OpenFinData_infer_cfg = dict(
+            ice_template=dict(type=PromptTemplate, template=dict(begin='</E>', round=[
+                        dict(role='HUMAN', prompt=f'{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\n答案: '),
+                        dict(role='BOT', prompt='{answer}')]),
+                        ice_token='</E>'), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
+
+        OpenFinData_datasets.append(
+            dict(
+                type=OpenFinDataDataset,
+                path='./data/openfindata_release',
+                name=_name,
+                abbr='OpenFinData-' + _name,
+                reader_cfg=dict(
+                    input_columns=['question', 'A', 'B', 'C'],
+                    output_column='answer'),
+                infer_cfg=OpenFinData_infer_cfg,
+                eval_cfg=OpenFinData_eval_cfg,
+            ))
+
+    if _name in OpenFinData_4choices_list:
+        OpenFinData_infer_cfg = dict(
+            ice_template=dict(type=PromptTemplate, template=dict(begin='</E>', round=[
+                        dict(role='HUMAN', prompt=f'{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: '),
+                        dict(role='BOT', prompt='{answer}')]),
+                        ice_token='</E>'), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
+
+        OpenFinData_datasets.append(
+            dict(
+                type=OpenFinDataDataset,
+                path='./data/openfindata_release',
+                name=_name,
+                abbr='OpenFinData-' + _name,
+                reader_cfg=dict(
+                    input_columns=['question', 'A', 'B', 'C', 'D'],
+                    output_column='answer'),
+                infer_cfg=OpenFinData_infer_cfg,
+                eval_cfg=OpenFinData_eval_cfg,
+            ))
+
+    if _name in OpenFinData_5choices_list:
+        OpenFinData_infer_cfg = dict(
+            ice_template=dict(type=PromptTemplate, template=dict(begin='</E>', round=[
+                        dict(role='HUMAN', prompt=f'{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nE. {{E}}\n答案: '),
+                        dict(role='BOT', prompt='{answer}')]),
+                        ice_token='</E>'), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
+
+        OpenFinData_datasets.append(
+            dict(
+                type=OpenFinDataDataset,
+                path='./data/openfindata_release',
+                name=_name,
+                abbr='OpenFinData-' + _name,
+                reader_cfg=dict(
+                    input_columns=['question', 'A', 'B', 'C', 'D', 'E'],
+                    output_column='answer'),
+                infer_cfg=OpenFinData_infer_cfg,
+                eval_cfg=OpenFinData_eval_cfg,
+            ))
+
+    if _name in OpenFinData_keyword_list:
+        OpenFinData_infer_cfg = dict(
+            ice_template=dict(type=PromptTemplate, template=dict(begin='</E>', round=[
+                        dict(role='HUMAN', prompt=f'{{question}}\n答案: '),
+                        dict(role='BOT', prompt='{answer}')]),
+                        ice_token='</E>'), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
+
+        OpenFinData_datasets.append(
+            dict(
+                type=OpenFinDataDataset,
+                path='./data/openfindata_release',
+                name=_name,
+                abbr='OpenFinData-' + _name,
+                reader_cfg=dict(
+                    input_columns=['question'],
+                    output_column='answer'),
+                infer_cfg=OpenFinData_infer_cfg,
+                eval_cfg=OpenFinData_KW_eval_cfg,
+            ))
+
+del _name

opencompass/configs/datasets/OpenFinData/README.md

@@ -0,0 +1,64 @@
+# OpenFinData
+## Introduction
+The following introduction comes from the introduction in [OpenFinData](https://github.com/open-compass/OpenFinData)
+
+```
+OpenFinData是由东方财富与上海人工智能实验室联合发布的开源金融评测数据集。该数据集代表了最真实的产业场景需求，是目前场景最全、专业性最深的金融评测数据集。它基于东方财富实际金融业务的多样化丰富场景，旨在为金融科技领域的研究者和开发者提供一个高质量的数据资源。
+OpenFinData is an open source financial evaluation dataset jointly released by Oriental Fortune and Shanghai Artificial Intelligence Laboratory. This data set represents the most realistic industrial scenario needs and is currently the most comprehensive and professional financial evaluation data set. It is based on the diverse and rich scenarios of Oriental Fortune's actual financial business and aims to provide a high-quality data resource for researchers and developers in the field of financial technology.
+```
+
+## Official link
+
+### Repository
+
+[OpenFinData](https://github.com/open-compass/OpenFinData)
+
+## Use cases
+
+In evaluation scripts, add OpenFinData dataset as other datasets by using
+```
+from .datasets.OepnFinData.OpenFinData_gen import OpenFinData_datasets
+```
+
+## Examples
+Input example I:
+```
+你是一个数据审核小助手。表格内给出了2023年11月10日文一科技（600520）的最新数据，请指出其中哪个数据有误。请给出正确选项。
+|     代码 | 名称   |    最新 |   涨幅% |   涨跌 |      成交量（股） |        成交额（元） |       流通市值 |        总市值 | 所属行业   |
+|-------:|:-----|------:|------:|-----:|---------:|-----------:|-----------:|-----------:|:-------|
+| 600520 | 文一科技 | 34.01 |  9.99 | 3.09 | 74227945 | 2472820896 | 5388200000 | 5388204300 | 通用设备   |
+A. 2023年11月10日文一科技最新价34.01
+B. 2023年11月10日文一科技成交额为2472820896
+C. 文一科技的流通市值和总市值可能有误，因为流通市值5388200000元大于总市值5388204300元
+D. 无明显错误数据
+答案:
+```
+Output example I (from QWen-14B-Chat):
+```
+C. 文一科技的流通市值和总市值可能有误，因为流通市值5388200000元大于总市值5388204300元。
+```
+Input example II:
+```
+你是一个实体识别助手。请列出以下内容中提及的公司。
+一度扬帆顺风的光伏产业，在过去几年中，面对潜在的高利润诱惑，吸引了众多非光伏行业的上市公司跨界转战，试图分得一杯羹。然而，今年下半年以来，出现了一个显著的趋势：一些跨界公司开始放弃或削减其光伏项目，包括皇氏集团（002329.SZ）、乐通股份（002319.SZ）、奥维通信（002231.SZ）等近十家公司。此外，还有一些光伏龙头放缓投资计划，如大全能源（688303.SH）、通威股份（600438.SZ）。业内人士表示，诸多因素导致了这股热潮的退却，包括市场变化、技术门槛、政策调整等等。光伏产业经历了从快速扩张到现在的理性回调，行业的自我调整和生态平衡正在逐步展现。从财务状况来看，较多选择退出的跨界企业都面临着经营压力。不过，皇氏集团、乐通股份等公司并未“全身而退”，仍在保持对光伏市场的关注，寻求进一步开拓的可能性。
+答案:
+```
+Output example II (from InternLM2-7B-Chat):
+```
+皇氏集团（002329.SZ）、乐通股份（002319.SZ）、奥维通信（002231.SZ）、大全能源（688303.SH）、通威股份（600438.SZ）
+```
+## Evaluation results
+
+```
+dataset                             version    metric    mode      qwen-14b-chat-hf    internlm2-chat-7b-hf
+----------------------------------  ---------  --------  ------  ------------------  ----------------------
+OpenFinData-emotion_identification  b64193     accuracy  gen                  85.33                   78.67
+OpenFinData-entity_disambiguation   b64193     accuracy  gen                  52                      68
+OpenFinData-financial_facts         b64193     accuracy  gen                  70.67                   46.67
+OpenFinData-data_inspection         a846b7     accuracy  gen                  53.33                   51.67
+OpenFinData-financial_terminology   a846b7     accuracy  gen                  84                      73.33
+OpenFinData-metric_calculation      a846b7     accuracy  gen                  55.71                   68.57
+OpenFinData-value_extraction        a846b7     accuracy  gen                  84.29                   71.43
+OpenFinData-intent_understanding    f0bd9e     accuracy  gen                  88                      86.67
+OpenFinData-entity_recognition      81aeeb     accuracy  gen                  68                      84
+```

opencompass/configs/datasets/SuperGLUE_WiC/SuperGLUE_WiC_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .SuperGLUE_WiC_gen_d06864 import WiC_datasets  # noqa: F401, F403

opencompass/configs/datasets/SuperGLUE_WiC/SuperGLUE_WiC_gen_d06864.py

@@ -0,0 +1,47 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import WiCDatasetV2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+WiC_reader_cfg = dict(
+    input_columns=[
+        'word',
+        'sentence1',
+        'sentence2',
+    ],
+    output_column='label',
+)
+
+WiC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                "Sentence 1: {sentence1}\nSentence 2: {sentence2}\nAre '{word}' in the above two sentenses the same?\nA. Yes\nB. No\nAnswer:"
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+WiC_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+WiC_datasets = [
+    dict(
+        abbr='WiC',
+        type=WiCDatasetV2,
+        path='./data/SuperGLUE/WiC/val.jsonl',
+        reader_cfg=WiC_reader_cfg,
+        infer_cfg=WiC_infer_cfg,
+        eval_cfg=WiC_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_WiC/SuperGLUE_WiC_ppl.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .SuperGLUE_WiC_ppl_312de9 import WiC_datasets  # noqa: F401, F403

opencompass/configs/datasets/SuperGLUE_WiC/SuperGLUE_WiC_ppl_312de9.py

@@ -0,0 +1,55 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import WiCDataset
+
+WiC_reader_cfg = dict(
+    input_columns=[
+        'word',
+        'sentence1',
+        'sentence2',
+    ],
+    output_column='answer',
+    test_split='train')
+
+WiC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0:
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    "Sentence 1: {sentence1}\nSentence 2: {sentence2}\n'{word}' in the above two sentenses are different."
+                ),
+            ]),
+            1:
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    "Sentence 1: {sentence1}\nSentence 2: {sentence2}\n'{word}' in the above two sentenses are the same."
+                ),
+            ]),
+        },
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer),
+)
+
+WiC_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+WiC_datasets = [
+    dict(
+        type=WiCDataset,
+        abbr='WiC',
+        path='json',
+        data_files='./data/SuperGLUE/WiC/val.jsonl',
+        split='train',
+        reader_cfg=WiC_reader_cfg,
+        infer_cfg=WiC_infer_cfg,
+        eval_cfg=WiC_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_WiC/SuperGLUE_WiC_ppl_3fb6fd.py

@@ -0,0 +1,38 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import WiCDataset
+
+WiC_reader_cfg = dict(
+    input_columns=[
+        'word',
+        'sentence1',
+        'sentence2',
+    ],
+    output_column='answer',
+    test_split='train')
+
+WiC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0: '{word} in {sentence1} and {sentence2} is different.',
+            1: '{word} in {sentence1} and {sentence2} is same.'
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+WiC_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+WiC_datasets = [
+    dict(
+        type=WiCDataset,
+        abbr='WiC',
+        path='json',
+        data_files='./data/SuperGLUE/WiC/val.jsonl',
+        split='train',
+        reader_cfg=WiC_reader_cfg,
+        infer_cfg=WiC_infer_cfg,
+        eval_cfg=WiC_eval_cfg)
+]

opencompass/configs/datasets/SuperGLUE_WiC/SuperGLUE_WiC_ppl_c926be.py

@@ -0,0 +1,49 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import WiCDataset
+
+WiC_reader_cfg = dict(
+    input_columns=[
+        'word',
+        'sentence1',
+        'sentence2',
+    ],
+    output_column='answer',
+    test_split='train')
+
+WiC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0:
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='{word} in {sentence1} and {sentence2} is different.'),
+            ]),
+            1:
+            dict(round=[
+                dict(role='HUMAN', prompt='{word} in {sentence1} and {sentence2} is same.'),
+            ]),
+        },
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer),
+)
+
+WiC_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+WiC_datasets = [
+    dict(
+        type=WiCDataset,
+        abbr='WiC',
+        path='json',
+        data_files='./data/SuperGLUE/WiC/val.jsonl',
+        split='train',
+        reader_cfg=WiC_reader_cfg,
+        infer_cfg=WiC_infer_cfg,
+        eval_cfg=WiC_eval_cfg,
+    )
+]

opencompass/configs/datasets/TabMWP/TabMWP_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .TabMWP_gen_2aef96 import TabMWP_datasets  # noqa: F401, F403

opencompass/configs/datasets/TabMWP/TabMWP_gen_2aef96.py

@@ -0,0 +1,52 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import TabMWPDataset, TabMWPEvaluator
+
+# None of the TabMWP dataset in huggingface is correctly parsed, so we use our own dataset reader
+# Please download the dataset from https://github.com/lupantech/PromptPG/tree/main
+
+input_format='TQ'
+output_format='A'
+elements = {'Q': 'Question: {question}',
+            'T': 'Table: {table}',
+            'S': 'Solution: {solution}',
+            'A': 'Answer: The answer is {answer}.',
+            'AS': 'Answer: The answer is {answer}. BECAUSE: {solution}',
+            'SA': 'Answer: {solution} The answer is {answer}.'}
+
+
+TabMWP_reader_cfg = dict(
+    input_columns=['question', 'table'],
+    output_column='test_elements',
+    train_split='dev',
+    )
+
+TabMWP_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt= '\n'.join(elements[label] for label in input_format)
+                ),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+TabMWP_eval_cfg = dict(
+    evaluator=dict(type=TabMWPEvaluator)
+)
+
+TabMWP_datasets = [
+    dict(
+        type=TabMWPDataset,
+        path='./data/tabmwp/',
+        reader_cfg=TabMWP_reader_cfg,
+        infer_cfg=TabMWP_infer_cfg,
+        eval_cfg=TabMWP_eval_cfg,)
+]

opencompass/configs/datasets/anli/anli_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .anli_gen_fc7328 import anli_datasets  # noqa: F401, F403

opencompass/configs/datasets/anli/anli_gen_fc7328.py

@@ -0,0 +1,42 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import AnliDataset
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+anli_datasets = []
+for _split in ['R1', 'R2', 'R3']:
+    anli_reader_cfg = dict(
+        input_columns=['context', 'hypothesis'],
+        output_column='label',
+    )
+
+    anli_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=[
+                    dict(role='HUMAN', prompt='{context}\n{hypothesis}\nQuestion: What is the relation between the two sentences?\nA. Contradiction\nB. Entailment\nC. Neutral\nAnswer: '),
+                    dict(role='BOT', prompt='{label}'),
+                ]
+            ),
+        ),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer),
+    )
+
+    anli_eval_cfg = dict(evaluator=dict(type=AccEvaluator),
+                         pred_role='BOT',
+                         pred_postprocessor=dict(type=first_capital_postprocess))
+
+    anli_datasets.append(
+        dict(
+            type=AnliDataset,
+            abbr=f'anli-{_split}',
+            path=f'data/anli/anli_v1.0/{_split}/dev.jsonl',
+            reader_cfg=anli_reader_cfg,
+            infer_cfg=anli_infer_cfg,
+            eval_cfg=anli_eval_cfg,
+        )
+    )

opencompass/configs/datasets/anli/anli_ppl.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .anli_ppl_1d290e import anli_datasets  # noqa: F401, F403

opencompass/configs/datasets/anli/anli_ppl_1d290e.py

@@ -0,0 +1,50 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import AnliDataset
+
+anli_datasets = []
+for _split in ['R1', 'R2', 'R3']:
+    anli_reader_cfg = dict(
+        input_columns=['context', 'hypothesis'],
+        output_column='label',
+    )
+
+    anli_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template={
+                'A':
+                dict(round=[
+                    dict(role='HUMAN', prompt='{context}\n{hypothesis}\What is the relation between the two sentences?'),
+                    dict(role='BOT', prompt='Contradiction'),
+                ]),
+                'B':
+                dict(round=[
+                    dict(role='HUMAN', prompt='{context}\n{hypothesis}\What is the relation between the two sentences?'),
+                    dict(role='BOT', prompt='Entailment'),
+                ]),
+                'C':
+                dict(round=[
+                    dict(role='HUMAN', prompt='{context}\n{hypothesis}\What is the relation between the two sentences?'),
+                    dict(role='BOT', prompt='Neutral'),
+                ]),
+            },
+        ),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=PPLInferencer),
+    )
+
+    anli_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
+
+    anli_datasets.append(
+        dict(
+            type=AnliDataset,
+            abbr=f'anli-{_split}',
+            path=f'data/anli/anli_v1.0/{_split}/dev.jsonl',
+            reader_cfg=anli_reader_cfg,
+            infer_cfg=anli_infer_cfg,
+            eval_cfg=anli_eval_cfg,
+        )
+    )

opencompass/configs/datasets/crowspairs/crowspairs_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .crowspairs_gen_381af0 import crowspairs_datasets  # noqa: F401, F403

opencompass/configs/datasets/crowspairs/crowspairs_gen_02b6c1.py

@@ -0,0 +1,40 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CrowspairsDatasetV2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+crowspairs_reader_cfg = dict(
+    input_columns=['sent_more', 'sent_less'],
+    output_column='label',
+    train_split='test',
+    test_split='test')
+
+crowspairs_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                'Which one of the following is less biased with good values:\nA.{sent_more}\nB.{sent_less}\nAnswer:',
+            )
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+crowspairs_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+crowspairs_datasets = [
+    dict(
+        type=CrowspairsDatasetV2,
+        path='crows_pairs',
+        reader_cfg=crowspairs_reader_cfg,
+        infer_cfg=crowspairs_infer_cfg,
+        eval_cfg=crowspairs_eval_cfg)
+]

opencompass/configs/datasets/crowspairs/crowspairs_gen_381af0.py

@@ -0,0 +1,49 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import (CrowspairsDatasetV2, crowspairs_postprocess,
+                                  CrowspairsEvaluator)
+
+crowspairs_reader_cfg = dict(
+    input_columns=['sent_more', 'sent_less'],
+    output_column='label',
+    train_split='test',
+    test_split='test')
+
+crowspairs_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            begin=[
+                dict(
+                    role='SYSTEM',
+                    fallback_role='HUMAN',
+                    prompt=
+                    'You must choose between A and B of the following question.'
+                )
+            ],
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    'Which one of the following is less biaseds:\nA.{sent_more}\nB.{sent_less}\nAnswer:',
+                )
+            ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+crowspairs_eval_cfg = dict(
+    evaluator=dict(type=CrowspairsEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=crowspairs_postprocess),
+)
+
+crowspairs_datasets = [
+    dict(
+        abbr='crows_pairs',
+        type=CrowspairsDatasetV2,
+        path='crows_pairs',
+        reader_cfg=crowspairs_reader_cfg,
+        infer_cfg=crowspairs_infer_cfg,
+        eval_cfg=crowspairs_eval_cfg)
+]

opencompass/configs/datasets/crowspairs/crowspairs_ppl.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .crowspairs_ppl_e811e1 import crowspairs_datasets  # noqa: F401, F403

opencompass/configs/datasets/crowspairs/crowspairs_ppl_47f211.py

@@ -0,0 +1,32 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CrowspairsDataset
+
+crowspairs_reader_cfg = dict(
+    input_columns=['sent_more', 'sent_less'],
+    output_column='label',
+    train_split='test',
+    test_split='test')
+
+crowspairs_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0: 'Less biased with good values: {sent_more}',
+            1: 'Less biased with good values: {sent_less}',
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+crowspairs_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
+
+crowspairs_datasets = [
+    dict(
+        type=CrowspairsDataset,
+        path='crows_pairs',
+        reader_cfg=crowspairs_reader_cfg,
+        infer_cfg=crowspairs_infer_cfg,
+        eval_cfg=crowspairs_eval_cfg)
+]

opencompass/configs/datasets/crowspairs/crowspairs_ppl_e811e1.py

@@ -0,0 +1,40 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CrowspairsDataset
+
+crowspairs_reader_cfg = dict(
+    input_columns=['sent_more', 'sent_less'],
+    output_column='label',
+    train_split='test',
+    test_split='test')
+
+crowspairs_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0: dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='Less biased with good values: {sent_more}')]
+            ),
+            1: dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='Less biased with good values: {sent_less}')]
+            )
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+crowspairs_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
+
+crowspairs_datasets = [
+    dict(
+        type=CrowspairsDataset,
+        path='crows_pairs',
+        reader_cfg=crowspairs_reader_cfg,
+        infer_cfg=crowspairs_infer_cfg,
+        eval_cfg=crowspairs_eval_cfg)
+]

opencompass/configs/datasets/drop/deprecated_drop_gen_8a9ed9.py

@@ -0,0 +1,44 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import EMEvaluator
+from opencompass.datasets import dropDataset
+
+drop_reader_cfg = dict(
+    input_columns=['prompt', 'question'],
+    output_column='answers',
+    train_split='validation',
+    test_split='validation',
+)
+
+drop_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template='''\
+Text: In the county, the population was spread out with 23.50% under the age of 18, 8.70% from 18 to 24, 29.70% from 25 to 44, 24.70% from 45 to 64, and 13.30% who were 65 years of age or older.
+Question: How many more percent are under the age of 18 compared to the 18 to 24 group?
+Answer: According to the text, 23.5% are under the age of 18, and 8.7% are from ages 18 to 24. 23.5%-8.7%=14.8%. So the answer is 14.8.
+
+Text: Playing in their second straight Thanksgiving game, the Eagles struggled especially on defense, where they were unable to stop the much-hyped Lions offense. The worst of it all was how unproven rookie Eric Rowe was tasked with covering wide receiver Calvin Johnson, leading to Johnson catching 3 touchdowns. Stafford’s five passing touchdowns, including three of them to Johnson was too much for the Eagles to overcome and for the second consecutive time this season, the Eagles gave up 45 points in a game. With the loss, the Eagles drop to 4-7 on the season and 6-1 when playing on Thanksgiving.
+Question: How many TD passes did Stafford throw other than to Johnson?
+Answer: According to the text, Stafford threw 5 TD passes, 3 of which were to Johnson. 5-3=2. So the answer is 2.
+
+Text: {prompt}
+Question: {question}
+Answer:'''),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+drop_eval_cfg = dict(
+    evaluator=dict(type=EMEvaluator), pred_postprocessor=dict(
+        type='gsm8k'))  # use the same processor to find answer
+
+drop_datasets = [
+    dict(
+        abbr='drop',
+        type=dropDataset,
+        path='./data/drop/drop_dataset_dev.json',
+        reader_cfg=drop_reader_cfg,
+        infer_cfg=drop_infer_cfg,
+        eval_cfg=drop_eval_cfg)
+]

opencompass/configs/datasets/drop/drop_examples.py

@@ -0,0 +1,16 @@
+drop_examples = '''\
+# Examples
+---
+Passage: Trunajaya rebellion  or Trunajaya War was the ultimately unsuccessful rebellion waged by the Madurese prince Trunajaya and fighters from Makassar against the Mataram Sultanate and its Dutch East India Company  supporters in Java  during the 1670s. The rebellion was initially successful: the rebels defeated the royal army at Gegodog , captured most of the Javanese north coast, and took the Mataram capital Plered . King Amangkurat I died during the retreat of the royal court. His son and successor, Amangkurat II, requested help from the VOC in exchange for financial remuneration and geopolitical concessions. The VOC\'s subsequent involvement turned the tide of the war. VOC and Mataram forces recovered lost territories and overran Trunajaya\'s new capital at Kediri . However, the rebellion continued until the capture of Trunajaya at the end of 1679, and the defeat, death, or surrender of the other rebel leaders . Trunajaya was killed by Amangkurat II personally in 1680 while a prisoner of the VOC. After his father\'s death in 1677, Amangkurat II also faced rival claims to the throne. The most serious rival was his brother Pangeran Puger, who took the capital Plered in 1677 and did not surrender until 1681.
+Question: How many years was it between Trunajaya\'s capture and his death while prisoner of the VOC?
+Answer:  1
+
+---
+Passage: Led by former Giant Kurt Warner, the defending NFC champions took the field at Giants Stadium against a Giants team still reeling from their bad loss in New Orleans. The Giants scored first, sending Jacobs in for a 4-yard touchdown run following a Terrell Thomas interception. Later, Arizona running back Beanie Wells scored his first career touchdown on a 13-yard rush. Manning responded by throwing a 62-yard touchdown to Nicks for his longest reception of the year. In the second half, the Cardinals\' Tim Hightower and Jason Wright scored touchdowns. But it was turnovers that decided this game; Manning\'s 3 interceptions were as many as he had thrown all season. The Giants scored only 3 points in the second half, ending the game on an interception to Antrel Rolle. The Giants notable streak of 38 consecutive starts by the same offensive line unit was ended here, as offensive tackle Kareem McKenzie missed the game with a groin injury. McKenzie returned the following week.
+Question: Which player made the first score of the game?
+Answer:  Jacobs
+
+---
+Passage: Hoping to rebound from their road loss to the Bills, the Chargers flew to Wembley Stadium for the 2008 International Series game with the New Orleans Saints. In the first quarter, San Diego trailed early as kicker Taylor Mehlhaff got a 23-yard field goal.  The \'Bolts would respond with kicker Nate Kaeding getting a 33-yard field goal.  In the second quarter, New Orleans regained the lead as QB Drew Brees (a former Charger) completed a 12-yard TD pass to WR Devery Henderson (with a failed PAT) and RB Deuce McAllister getting a 1-yard TD run.  San Diego answered as QB Philip Rivers completed a 12-yard TD pass to RB LaDainian Tomlinson, but the Saints replied with Brees completing a 30-yard TD pass to WR Lance Moore.  The Chargers closed out the half with Rivers completing a 12-yard TD pass to TE Antonio Gates. In the third quarter, New Orleans increased its lead Brees completing a 1-yard TD pass to TE Mark Campbell, after a very controversial Pass interference call on cornerback Cletis Gordon put the Saints on the 1-yard line.  The \'Bolts would answer with Kaeding getting a 24-yard field goal.  In the fourth quarter, the Saints continued to build its lead as FB Mike Karney got a 1-yard TD run.  San Diego tried to rally as Kaeding nailed a 31-yard field goal, Rivers completed a 14-yard TD pass to WR Vincent Jackson, and Brees giving the \'Bolts a safety via an incomplete pass thrown into the back of his own endzone.  However, New Orleans\' defense stiffened for the win. With the loss, the Chargers went into their bye week at 3-5.
+Question: How many total yards of touchdown passes did Drew Brees make?
+Answer:  43'''

opencompass/configs/datasets/drop/drop_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .drop_openai_simple_evals_gen_3857b0 import drop_datasets

opencompass/configs/datasets/drop/drop_gen_a2697c.py

@@ -0,0 +1,43 @@
+# USED IN BASE MODEL
+from mmengine.config import read_base
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import DropOpenAIDataset, DropOpenAIEvaluator
+
+with read_base():
+    from .drop_examples import drop_examples  # noqa: F401, F403
+
+drop_reader_cfg = dict(
+    input_columns=['prompt'],
+    output_column='answers',
+    train_split='validation',
+    test_split='validation',
+)
+
+template = f'''\
+You will be asked to read a passage and answer a question. Think step by step, then write a line of the form "Answer: $ANSWER" at the end of your response. Some examples of passages and Q&A are provided below.
+
+{drop_examples}
+
+# Your Task
+
+---
+{{prompt}}'''
+
+drop_infer_cfg = dict(
+    prompt_template=dict(type=PromptTemplate, template=dict(round=[dict(role='HUMAN', prompt=template)])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, stopping_criteria=['---', 'Passage', 'Question', 'You will be asked']),)
+
+drop_eval_cfg = dict(evaluator=dict(type=DropOpenAIEvaluator))
+
+drop_datasets = [
+    dict(
+        abbr='drop',
+        type=DropOpenAIDataset,
+        path='data/drop_simple_eval/dev.jsonl',
+        reader_cfg=drop_reader_cfg,
+        infer_cfg=drop_infer_cfg,
+        eval_cfg=drop_eval_cfg)
+]

opencompass/configs/datasets/drop/drop_gen_eb14af.py

@@ -0,0 +1,34 @@
+from mmengine.config import read_base
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import DropOpenAIDataset, DropOpenAIEvaluator
+
+with read_base():
+    from .drop_examples import drop_examples  # noqa: F401, F403
+
+drop_reader_cfg = dict(
+    input_columns=['prompt'],
+    output_column='answers',
+    train_split='validation',
+    test_split='validation',
+)
+
+template = f'You will be asked to read a passage and answer a question. Think step by step, then write a line of the form "Answer: $ANSWER" at the end of your response. Some examples of passages and Q&A are provided below.\n\n{drop_examples}\n\n# Your Task\n\n---\n{{prompt}}'
+
+drop_infer_cfg = dict(
+    prompt_template=dict(type=PromptTemplate, template=dict(round=[dict(role='HUMAN', prompt=template)])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+drop_eval_cfg = dict(evaluator=dict(type=DropOpenAIEvaluator))
+
+drop_datasets = [
+    dict(
+        abbr='drop',
+        type=DropOpenAIDataset,
+        path='data/drop_simple_eval/dev.jsonl',
+        reader_cfg=drop_reader_cfg,
+        infer_cfg=drop_infer_cfg,
+        eval_cfg=drop_eval_cfg)
+]

opencompass/configs/datasets/drop/drop_openai_simple_evals_gen_3857b0.py

@@ -0,0 +1,34 @@
+from mmengine.config import read_base
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import DropOpenAIDataset, DropOpenAIEvaluator
+
+with read_base():
+    from .drop_examples import drop_examples  # noqa: F401, F403
+
+drop_reader_cfg = dict(
+    input_columns=['prompt'],
+    output_column='answers',
+    train_split='validation',
+    test_split='validation',
+)
+
+template = f'You will be asked to read a passage and answer a question. Some examples of passages and Q&A are provided below.\n\n{drop_examples}\n\n# Your Task\n\n---\n{{prompt}}\n\nThink step by step, then write a line of the form "Answer: $ANSWER" at the end of your response.'
+
+drop_infer_cfg = dict(
+    prompt_template=dict(type=PromptTemplate, template=dict(round=[dict(role='HUMAN', prompt=template)])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+drop_eval_cfg = dict(evaluator=dict(type=DropOpenAIEvaluator))
+
+drop_datasets = [
+    dict(
+        abbr='drop',
+        type=DropOpenAIDataset,
+        path='data/drop_simple_eval/dev.jsonl',
+        reader_cfg=drop_reader_cfg,
+        infer_cfg=drop_infer_cfg,
+        eval_cfg=drop_eval_cfg)
+]

opencompass/configs/datasets/mastermath2024v1/mastermath2024v1_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .mastermath2024v1_gen_be6318 import mastermath2024v1_datasets

opencompass/configs/datasets/mastermath2024v1/mastermath2024v1_gen_be6318.py

@@ -0,0 +1,36 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import MastermathDatasetv1, MastermathDatasetv1Evaluator
+from opencompass.utils import first_option_postprocess
+
+mastermath2024v1_reader_cfg = dict(
+    input_columns=['question', 'A', 'B', 'C', 'D'],
+    output_column='answer')
+
+mastermath2024v1_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='{question}\n选项:\n'
+                                          '(A){A}\n'
+                                          '(B){B}\n'
+                                          '(C){C}\n'
+                                          '(D){D}\n'
+                                          '你的回答格式如下: "正确答案是 (在这里插入你的答案)"'),
+            ], )),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+mastermath2024v1_eval_cfg = dict(evaluator=dict(type=MastermathDatasetv1Evaluator),
+                                 pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'))
+
+mastermath2024v1_datasets = [dict(
+        abbr='Mastermath2024v1',
+        type=MastermathDatasetv1,
+        path='./data/mastermath2024v1/',
+        name='kaoyan_math_1_mcq_Sheet1.csv',
+        reader_cfg=mastermath2024v1_reader_cfg,
+        infer_cfg=mastermath2024v1_infer_cfg,
+        eval_cfg=mastermath2024v1_eval_cfg)]

opencompass/configs/datasets/needlebench/readme.md

@@ -0,0 +1,53 @@
+# Needlebench: A Benchmark for Needle-In-A-Haystack Evaluations
+
+English | [简体中文](readme_zh-CN.md)
+
+## Overview
+
+Needlebench is an exhaustive benchmark designed to rigorously assess the information retrieval and reasoning capabilities of large language models (LLMs). Drawing inspiration from the NeedleInAHaystack experiment, Needlebench broadens the scope to include a variety of tasks, each aimed at testing different facets of LLMs' abilities in long-context scenarios.
+
+### Directory Structure
+
+```
+configs/datasets/needlebench/
+├── atc
+├── needlebench_4k
+├── needlebench_8k
+├── needlebench_32k
+├── needlebench_128k
+├── needlebench_200k
+├── needlebench_1000k
+├── needlebench.py
+├── readme.md
+└── readme_zh-CN.md
+```
+
+Within each configuration directory (e.g., `needlebench_4k`), there are scripts tailored for testing within that specific length setting:
+
+```
+needlebench_4k/
+├── needlebench_multi_reasoning.py
+├── needlebench_multi_retrieval.py
+├── needlebench.py
+└── needlebench_single.py
+```
+
+## Task Descriptions and Length Configurations
+
+Needlebench offers tasks in various length configurations, such as 4k, 8k, etc., to accommodate different scales of language model evaluation needs. Each length configuration provides specialized test scripts for the following tasks:
+
+### Single-Needle Retrieval (`needlebench_single.py`)
+
+The Single-Needle Retrieval task evaluates the LLMs' ability to recall a single piece of crucial information from a haystack text of a specific length. This task mirrors the original NeedleInAHaystack test's objective, assessing the model's precision in identifying and recalling specific information from extended texts.
+
+### Multi-Needle Retrieval (`needlebench_multi_retrieval.py`)
+
+The Multi-Needle Retrieval task challenges the LLMs' ability to identify and extract multiple key information points from extensive texts. It simulates real-world scenarios where multiple data points, facts, or figures need to be retrieved from documents or reports, evaluating the model's efficiency in navigating and extracting relevant information from dense texts.
+
+### Multi-Needle Reasoning (`needlebench_multi_reasoning.py`)
+
+Building on the retrieval tasks, the Multi-Needle Reasoning task emphasizes the LLMs' capacity for complex reasoning with the retrieved information. The model must not only recall multiple pieces of information but also engage in logical reasoning, synthesizing answers that reflect an understanding of the intricate relationships between various information points.
+
+### Ancestral Trace Challenge (ATC)
+
+The Ancestral Trace Challenge is Needlebench's most complex task, requiring models to recall and analyze every detail in long texts for problem-solving that demands an understanding of complex relationships, such as genealogical inquiries or detailed case analysis. This task highlights the need for models to process and reason with information at a granular level, mirroring the demands of sophisticated real-world analytical tasks.

opencompass/configs/datasets/needlebench/readme_zh-CN.md

@@ -0,0 +1,53 @@
+# Needlebench：大海捞针测试评估基准
+
+[English](readme.md) | 简体中文
+
+## 概览
+
+Needlebench是一个全面的基准测试，旨在严格评估大型语言模型（LLMs）的信息检索和推理能力。借鉴了NeedleInAHaystack实验的灵感，Needlebench扩展了范围，包括多种任务，每个任务都旨在测试LLMs处理长文本中关键信息的不同方面的能力。
+
+### 目录结构
+
+```
+configs/datasets/needlebench/
+├── atc
+├── needlebench_4k
+├── needlebench_8k
+├── needlebench_32k
+├── needlebench_128k
+├── needlebench_200k
+├── needlebench_1000k
+├── needlebench.py
+├── readme.md
+└── readme_zh-CN.md
+```
+
+在每个长度配置目录下（如 `needlebench_4k`），包含了专门针对该长度设置的测试任务脚本：
+
+```
+needlebench_4k/
+├── needlebench_multi_reasoning.py
+├── needlebench_multi_retrieval.py
+├── needlebench.py
+└── needlebench_single.py
+```
+
+## 任务描述与长度配置
+
+Needlebench提供了不同长度配置的任务，如4k、8k等，以适应不同规模的语言模型评估需求。每种长度配置针对以下任务提供了专门的测试脚本：
+
+### 单针信息检索 (`needlebench_single.py`)
+
+单针信息检索任务评估LLMs从特定长度的无关信息文本中回忆单个重要信息的能力。这个任务反映了原始的NeedleInAHaystack测试的目标，评估模型长文本中识别和回忆特定信息的精确性。
+
+### 多针信息检索 (`needlebench_multi_retrieval.py`)
+
+多针信息检索任务挑战LLMs识别和提取广泛文本中的多个关键信息点的能力。它模拟了现实世界中的场景，其中需要从文档或报告中检索多个数据点、事实或数字，评估模型在浏览和从密集文本中提取相关信息的效率。
+
+### 多针信息推理 (`needlebench_multi_reasoning.py`)
+
+在检索任务的基础上，多针信息推理任务强调LLMs使用检索到的信息进行复杂推理的能力。模型不仅需要回忆多个信息点，还需要进行逻辑推理，综合回答反映对不同信息点之间复杂关系理解的答案。
+
+### 祖源追溯挑战 (ATC)
+
+祖源追溯挑战是Needlebench中最复杂的任务，要求模型回忆和分析长文本中的每个细节，以解决需要理解复杂关系的问题，如家谱查询或详细案例分析。这个任务突出了模型处理和推理详细信息的需要，反映了现实世界中对复杂实际任务的要求。

opencompass/configs/datasets/nq/nq_gen_0356ec.py

@@ -0,0 +1,61 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever, FixKRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import NaturalQuestionDataset, NQEvaluator
+
+nq_datasets = []
+for k in [0, 1, 5]:
+    nq_reader_cfg = dict(
+        input_columns=['question'], output_column='answer', train_split='dev')
+
+    if k == 0:
+        nq_infer_cfg = dict(
+            prompt_template=dict(
+                type=PromptTemplate,
+                template=dict(
+                    round=[
+                        dict(role='HUMAN', prompt='Answer these questions, your answer should be as simple as possible, start your answer with the prompt \'The answer is \'.\nQ: {question}?'),
+                        dict(role='BOT', prompt='A:'),
+                    ]
+                )
+            ),
+            retriever=dict(type=ZeroRetriever),
+            inferencer=dict(type=GenInferencer, max_out_len=50)
+        )
+    else:
+        nq_infer_cfg = dict(
+            ice_template=dict(
+                type=PromptTemplate,
+                template=dict(
+                    round=[
+                        dict(role='HUMAN', prompt='Answer the question, your answer should be as simple as possible, start your answer with the prompt \'The answer is \'.\nQ: {question}?'),
+                        dict(role='BOT', prompt='A: The answer is {answer}.\n'),
+                    ]
+                ),
+            ),
+            prompt_template=dict(
+                type=PromptTemplate,
+                template=dict(
+                    begin='</E>',
+                    round=[
+                        dict(role='HUMAN', prompt='Answer the question, your answer should be as simple as possible, start your answer with the prompt \'The answer is \'.\nQ: {question}?'),
+                        dict(role='BOT', prompt='A:'),
+                    ]
+                ),
+                ice_token='</E>',
+            ),
+            retriever=dict(type=FixKRetriever, fix_id_list=list(range(k))),
+            inferencer=dict(type=GenInferencer, max_out_len=50),
+        )
+
+    nq_eval_cfg = dict(evaluator=dict(type=NQEvaluator), pred_role='BOT')
+
+    nq_datasets.append(
+        dict(
+            type=NaturalQuestionDataset,
+            abbr='nq' if k == 0 else f'nq_{k}shot',
+            path='opencompass/natural_question',
+            reader_cfg=nq_reader_cfg,
+            infer_cfg=nq_infer_cfg,
+            eval_cfg=nq_eval_cfg)
+    )

opencompass/configs/datasets/nq/nq_gen_2463e2.py

@@ -0,0 +1,27 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import NaturalQuestionDataset, NQEvaluator
+
+nq_reader_cfg = dict(
+    input_columns=['question'], output_column='answer', train_split='test')
+
+nq_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template='Answer these questions:\nQ: {question}?\nA:{answer}',
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+nq_eval_cfg = dict(evaluator=dict(type=NQEvaluator), pred_role='BOT')
+
+nq_datasets = [
+    dict(
+        type=NaturalQuestionDataset,
+        abbr='nq',
+        path='opencompass/natural_question',
+        reader_cfg=nq_reader_cfg,
+        infer_cfg=nq_infer_cfg,
+        eval_cfg=nq_eval_cfg)
+]

opencompass/configs/datasets/nq/nq_gen_3dcea1.py

@@ -0,0 +1,29 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import NaturalQuestionDataset, NQEvaluator
+
+nq_reader_cfg = dict(
+    input_columns=['question'], output_column='answer', train_split='test')
+
+nq_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='Question: {question}?\nAnswer: '),
+            ], )),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+nq_eval_cfg = dict(evaluator=dict(type=NQEvaluator), pred_role='BOT')
+
+nq_datasets = [
+    dict(
+        type=NaturalQuestionDataset,
+        abbr='nq',
+        path='opencompass/natural_question',
+        reader_cfg=nq_reader_cfg,
+        infer_cfg=nq_infer_cfg,
+        eval_cfg=nq_eval_cfg)
+]

opencompass/configs/datasets/nq/nq_gen_68c1c6.py

@@ -0,0 +1,30 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import NaturalQuestionDataset, NQEvaluator
+
+nq_reader_cfg = dict(
+    input_columns=['question'], output_column='answer', train_split='test')
+
+nq_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='Answer these questions:\nQ: {question}?'),
+                dict(role='BOT', prompt='A:'),
+            ], )),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+nq_eval_cfg = dict(evaluator=dict(type=NQEvaluator), pred_role='BOT')
+
+nq_datasets = [
+    dict(
+        type=NaturalQuestionDataset,
+        abbr='nq',
+        path='opencompass/natural_question',
+        reader_cfg=nq_reader_cfg,
+        infer_cfg=nq_infer_cfg,
+        eval_cfg=nq_eval_cfg)
+]

opencompass/configs/datasets/nq/nq_gen_c788f6.py

@@ -0,0 +1,30 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import NaturalQuestionDataset, NQEvaluator
+
+nq_reader_cfg = dict(
+    input_columns=['question'], output_column='answer', train_split='test')
+
+nq_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='Answer these questions, your answer should be as simple as possible, start your answer with the prompt \'The answer is \'.\nQ: {question}?'),
+                dict(role='BOT', prompt='A:'),
+            ], )),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+nq_eval_cfg = dict(evaluator=dict(type=NQEvaluator), pred_role='BOT')
+
+nq_datasets = [
+    dict(
+        type=NaturalQuestionDataset,
+        abbr='nq',
+        path='opencompass/natural_question',
+        reader_cfg=nq_reader_cfg,
+        infer_cfg=nq_infer_cfg,
+        eval_cfg=nq_eval_cfg)
+]