Add files using upload-large-folder tool

opencompass/configs/datasets/CLUE_CMRC/CLUE_CMRC_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .CLUE_CMRC_gen_1bd3c8 import CMRC_datasets  # noqa: F401, F403

opencompass/configs/datasets/CLUE_CMRC/CLUE_CMRC_gen_1bd3c8.py

@@ -0,0 +1,35 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import EMEvaluator
+from opencompass.datasets import CMRCDataset, cmrc_postprocess
+
+CMRC_reader_cfg = dict(
+    input_columns=['question', 'context'], output_column='answers')
+
+CMRC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='根据文章回答问题。你的答案应该尽可能简练，请以 ‘答案是’ 开头的句式作答。\n文章：{context}\n问：{question}\n答：'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+CMRC_eval_cfg = dict(
+    evaluator=dict(type=EMEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=cmrc_postprocess),
+)
+
+CMRC_datasets = [
+    dict(
+        type=CMRCDataset,
+        abbr='CMRC_dev',
+        path='opencompass/cmrc_dev',
+        reader_cfg=CMRC_reader_cfg,
+        infer_cfg=CMRC_infer_cfg,
+        eval_cfg=CMRC_eval_cfg),
+]

opencompass/configs/datasets/CLUE_CMRC/CLUE_CMRC_gen_3749cd.py

@@ -0,0 +1,33 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import EMEvaluator
+from opencompass.datasets import CMRCDataset
+
+CMRC_reader_cfg = dict(
+    input_columns=['question', 'context'], output_column='answers')
+
+CMRC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(role='HUMAN', prompt='文章：{context}\n根据上文，回答如下问题：{question}'),
+            dict(role='BOT', prompt='答：'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+CMRC_eval_cfg = dict(
+    evaluator=dict(type=EMEvaluator),
+    pred_role='BOT',
+)
+
+CMRC_datasets = [
+    dict(
+        type=CMRCDataset,
+        abbr='CMRC_dev',
+        path='opencompass/cmrc_dev',
+        reader_cfg=CMRC_reader_cfg,
+        infer_cfg=CMRC_infer_cfg,
+        eval_cfg=CMRC_eval_cfg),
+]

opencompass/configs/datasets/CLUE_CMRC/CLUE_CMRC_gen_8484b9.py

@@ -0,0 +1,27 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import EMEvaluator
+from opencompass.datasets import CMRCDataset
+
+CMRC_reader_cfg = dict(
+    input_columns=['question', 'context'], output_column='answers')
+
+CMRC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template='文章：{context}\n根据上文，回答如下问题： {question}\n答：'),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+CMRC_eval_cfg = dict(evaluator=dict(type=EMEvaluator), )
+
+CMRC_datasets = [
+    dict(
+        type=CMRCDataset,
+        abbr='CMRC_dev',
+        path='opencompass/cmrc_dev',
+        reader_cfg=CMRC_reader_cfg,
+        infer_cfg=CMRC_infer_cfg,
+        eval_cfg=CMRC_eval_cfg),
+]

opencompass/configs/datasets/CLUE_CMRC/CLUE_CMRC_gen_941108.py

@@ -0,0 +1,34 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import EMEvaluator
+from opencompass.datasets import CMRCDataset
+
+CMRC_reader_cfg = dict(
+    input_columns=['question', 'context'], output_column='answers')
+
+CMRC_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='文章：{context}\n根据上文，回答如下问题：\n{question}\n答：'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+CMRC_eval_cfg = dict(
+    evaluator=dict(type=EMEvaluator),
+    pred_role='BOT',
+)
+
+CMRC_datasets = [
+    dict(
+        type=CMRCDataset,
+        abbr='CMRC_dev',
+        path='opencompass/cmrc_dev',
+        reader_cfg=CMRC_reader_cfg,
+        infer_cfg=CMRC_infer_cfg,
+        eval_cfg=CMRC_eval_cfg),
+]

opencompass/configs/datasets/FewCLUE_eprstmt/FewCLUE_eprstmt_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .FewCLUE_eprstmt_gen_740ea0 import eprstmt_datasets  # noqa: F401, F403

opencompass/configs/datasets/FewCLUE_eprstmt/FewCLUE_eprstmt_gen_740ea0.py

@@ -0,0 +1,49 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import EprstmtDatasetV2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+eprstmt_reader_cfg = dict(
+    input_columns=['sentence'], output_column='label', test_split='train')
+
+eprstmt_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                '内容： "{sentence}"。请对上述内容进行情绪分类。\nA. 积极\nB. 消极\n请从”A“，”B“中进行选择。\n答：'
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+eprstmt_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+eprstmt_datasets = [
+    dict(
+        abbr='eprstmt-dev',
+        type=EprstmtDatasetV2,
+        path='./data/FewCLUE/eprstmt/dev_few_all.json',
+        reader_cfg=eprstmt_reader_cfg,
+        infer_cfg=eprstmt_infer_cfg,
+        eval_cfg=eprstmt_eval_cfg,
+    ),
+    dict(
+        abbr='eprstmt-test',
+        type=EprstmtDatasetV2,
+        path='./data/FewCLUE/eprstmt/test_public.json',
+        reader_cfg=eprstmt_reader_cfg,
+        infer_cfg=eprstmt_infer_cfg,
+        eval_cfg=eprstmt_eval_cfg,
+    ),
+]

opencompass/configs/datasets/FewCLUE_eprstmt/FewCLUE_eprstmt_ppl.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .FewCLUE_eprstmt_ppl_f1e631 import eprstmt_datasets  # noqa: F401, F403

opencompass/configs/datasets/FewCLUE_eprstmt/FewCLUE_eprstmt_ppl_1ce587.py

@@ -0,0 +1,41 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HFDataset
+
+eprstmt_reader_cfg = dict(
+    input_columns=['sentence'], output_column='label', test_split='train')
+
+eprstmt_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'Negative': ' 内容： "{sentence}"。情绪分类：消极。',
+            'Positive': ' 内容： "{sentence}"。情绪分类：积极。',
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+eprstmt_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+eprstmt_datasets = [
+    dict(
+        type=HFDataset,
+        abbr='eprstmt-dev',
+        path='json',
+        data_files='./data/FewCLUE/eprstmt/dev_few_all.json',
+        split='train',
+        reader_cfg=eprstmt_reader_cfg,
+        infer_cfg=eprstmt_infer_cfg,
+        eval_cfg=eprstmt_eval_cfg),
+    dict(
+        type=HFDataset,
+        abbr='eprstmt-test',
+        path='json',
+        data_files='./data/FewCLUE/eprstmt/test_public.json',
+        split='train',
+        reader_cfg=eprstmt_reader_cfg,
+        infer_cfg=eprstmt_infer_cfg,
+        eval_cfg=eprstmt_eval_cfg)
+]

opencompass/configs/datasets/FewCLUE_eprstmt/FewCLUE_eprstmt_ppl_f1e631.py

@@ -0,0 +1,49 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HFDataset
+
+eprstmt_reader_cfg = dict(
+    input_columns=['sentence'], output_column='label', test_split='train')
+
+eprstmt_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'Negative':
+            dict(round=[
+                dict(role='HUMAN', prompt='内容： "{sentence}"。情绪分类：'),
+                dict(role='BOT', prompt='消极。')
+            ]),
+            'Positive':
+            dict(round=[
+                dict(role='HUMAN', prompt='内容： "{sentence}"。情绪分类：'),
+                dict(role='BOT', prompt='积极。')
+            ]),
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+eprstmt_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+eprstmt_datasets = [
+    dict(
+        type=HFDataset,
+        abbr='eprstmt-dev',
+        path='json',
+        data_files='./data/FewCLUE/eprstmt/dev_few_all.json',
+        split='train',
+        reader_cfg=eprstmt_reader_cfg,
+        infer_cfg=eprstmt_infer_cfg,
+        eval_cfg=eprstmt_eval_cfg),
+    dict(
+        type=HFDataset,
+        abbr='eprstmt-test',
+        path='json',
+        data_files='./data/FewCLUE/eprstmt/test_public.json',
+        split='train',
+        reader_cfg=eprstmt_reader_cfg,
+        infer_cfg=eprstmt_infer_cfg,
+        eval_cfg=eprstmt_eval_cfg)
+]

opencompass/configs/datasets/PJExam/PJExam_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .PJExam_gen_8cd97c import PJExam_datasets  # noqa: F401, F403

opencompass/configs/datasets/PJExam/PJExam_gen_8cd97c.py

@@ -0,0 +1,54 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import PJExamDataset, PJExamEvaluator
+
+PJExam_datasets = []
+for _name in [
+        'gk-2022-v1', 'gk-2022-v1-math', 'gk-2023-v1', 'gk-2023-v1-math',
+        'gk-2023-v2', 'gk-2023-v2-math', 'zk-2022-v1'
+]:
+    _hint = '请你做一道</major>选择题\n请你一步一步思考并将思考过程写在【解析】和<eoe>之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和<eoa>之间。\n例如：【答案】A<eoa>\n完整的题目回答的格式如下：\n【解析】...<eoe>\n【答案】...<eoa>\n请你严格按照上述格式作答。\n题目如下：\n'
+    _reader_cfg = {
+        'input_columns': ['question'],
+        'output_column': 'std_ans',
+    },
+    _infer_cfg = {
+        'ice_template': {
+            'type': PromptTemplate,
+            'template': {
+                'round': [{
+                    'role': 'HUMAN',
+                    'prompt': _hint + '{question}',
+                }]
+            },
+            'ice_token': '</E>'
+        },
+        'retriever': {
+            'type': ZeroRetriever
+        },
+        'inferencer': {
+            'type': GenInferencer,
+            'max_out_len': 1024,
+        }
+    }
+    _eval_cfg = {
+        'evaluator': {
+            'type': PJExamEvaluator
+        },
+        'pred_role': 'BOT',
+        'ds_column': 'eval_infos'
+    }
+    _dataset = {
+        'type': PJExamDataset,
+        'abbr': 'PJExamDataset-' + _name,
+        'path': './data/PJExam',
+        'name': _name,
+        'reader_cfg': _reader_cfg,
+        'infer_cfg': _infer_cfg,
+        'eval_cfg': _eval_cfg,
+    }
+
+    PJExam_datasets.append(_dataset)
+
+del _name, _hint, _reader_cfg, _infer_cfg, _eval_cfg, _dataset

opencompass/configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .SuperGLUE_COPA_gen_91ca53 import COPA_datasets  # noqa: F401, F403

opencompass/configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_gen_91ca53.py

@@ -0,0 +1,44 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import COPADatasetV2
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+COPA_reader_cfg = dict(
+    input_columns=['question', 'premise', 'choice1', 'choice2'],
+    output_column='label',
+)
+
+COPA_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    '{premise}\nQuestion: Which may be the {question}?\nA. {choice1}\nB. {choice2}\nAnswer:'
+                ),
+            ], ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+COPA_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_option_postprocess, options='AB'),
+)
+
+COPA_datasets = [
+    dict(
+        abbr='COPA',
+        type=COPADatasetV2,
+        path='./data/SuperGLUE/COPA/val.jsonl',
+        reader_cfg=COPA_reader_cfg,
+        infer_cfg=COPA_infer_cfg,
+        eval_cfg=COPA_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_ppl.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .SuperGLUE_COPA_ppl_9f3618 import COPA_datasets  # noqa: F401, F403

opencompass/configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_ppl_54058d.py

@@ -0,0 +1,34 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HFDataset
+
+COPA_reader_cfg = dict(
+    input_columns=['question', 'premise', 'choice1', 'choice2'],
+    output_column='label',
+    test_split='train')
+
+COPA_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0: 'Premise:{premise}。\nQuestion:{question}。\nAnswer: {choice1}.',
+            1: 'Passage:{premise}。\nQuestion:{question}。\nAnswer: {choice2}.',
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+COPA_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+COPA_datasets = [
+    dict(
+        type=HFDataset,
+        abbr='COPA',
+        path='json',
+        data_files='./data/SuperGLUE/COPA/val.jsonl',
+        split='train',
+        reader_cfg=COPA_reader_cfg,
+        infer_cfg=COPA_infer_cfg,
+        eval_cfg=COPA_eval_cfg)
+]

opencompass/configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_ppl_5c24f1.py

@@ -0,0 +1,45 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HFDataset
+
+COPA_reader_cfg = dict(
+    input_columns=['question', 'premise', 'choice1', 'choice2'],
+    output_column='label',
+    test_split='train')
+
+COPA_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0:
+            dict(round=[
+                dict(role='HUMAN', prompt='{premise}\nQuestion: {question}\nAnswer:'),
+                dict(role='BOT', prompt='{choice1}'),
+            ]),
+            1:
+            dict(round=[
+                dict(role='HUMAN', prompt='{premise}\nQuestion: {question}\nAnswer:'),
+                dict(role='BOT', prompt='{choice2}'),
+            ]),
+        },
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer),
+)
+
+COPA_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+COPA_datasets = [
+    dict(
+        type=HFDataset,
+        abbr='COPA',
+        path='json',
+        data_files='./data/SuperGLUE/COPA/val.jsonl',
+        split='train',
+        reader_cfg=COPA_reader_cfg,
+        infer_cfg=COPA_infer_cfg,
+        eval_cfg=COPA_eval_cfg,
+    )
+]

opencompass/configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_ppl_9f3618.py

@@ -0,0 +1,49 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HFDataset
+
+COPA_reader_cfg = dict(
+    input_columns=['question', 'premise', 'choice1', 'choice2'],
+    output_column='label',
+    test_split='train')
+
+COPA_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0:
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='{premise}\nQuestion: What may be the {question}?\nAnswer:'),
+                dict(role='BOT', prompt='{choice1}'),
+            ]),
+            1:
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='{premise}\nQuestion: What may be the {question}?\nAnswer:'),
+                dict(role='BOT', prompt='{choice2}'),
+            ]),
+        },
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer),
+)
+
+COPA_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+COPA_datasets = [
+    dict(
+        type=HFDataset,
+        abbr='COPA',
+        path='json',
+        data_files='./data/SuperGLUE/COPA/val.jsonl',
+        split='train',
+        reader_cfg=COPA_reader_cfg,
+        infer_cfg=COPA_infer_cfg,
+        eval_cfg=COPA_eval_cfg,
+    )
+]

opencompass/configs/datasets/adv_glue/__init__.py

@@ -0,0 +1,11 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .adv_glue_sst2.adv_glue_sst2_gen import adv_sst2_datasets
+    from .adv_glue_qqp.adv_glue_qqp_gen import adv_qqp_datasets
+    from .adv_glue_rte.adv_glue_rte_gen import adv_rte_datasets
+    from .adv_glue_qnli.adv_glue_qnli_gen import adv_qnli_datasets
+    from .adv_glue_mnli.adv_glue_mnli_gen import adv_mnli_datasets
+    from .adv_glue_mnli_mm.adv_glue_mnli_mm_gen import adv_mnli_mm_datasets
+
+datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])

opencompass/configs/datasets/collections/base_core.py

@@ -0,0 +1,20 @@
+from mmengine.config import read_base
+
+with read_base():
+    from ..mmlu.mmlu_ppl_ac766d import mmlu_datasets
+    from ..cmmlu.cmmlu_ppl_041cbf import cmmlu_datasets
+    from ..ceval.ceval_ppl_1cd8bf import ceval_datasets
+    from ..GaokaoBench.GaokaoBench_no_subjective_gen_d21e37 import GaokaoBench_datasets
+    from ..triviaqa.triviaqa_wiki_1shot_gen_20a989 import triviaqa_datasets
+    from ..nq.nq_open_1shot_gen_20a989 import nq_datasets
+    from ..race.race_ppl_abed12 import race_datasets
+    from ..winogrande.winogrande_5shot_ll_252f01 import winogrande_datasets
+    from ..hellaswag.hellaswag_10shot_ppl_59c85e import hellaswag_datasets
+    from ..bbh.bbh_gen_98fba6 import bbh_datasets
+    from ..gsm8k.gsm8k_gen_ee684f import gsm8k_datasets
+    from ..math.math_evaluatorv2_gen_2f4a71 import math_datasets
+    from ..TheoremQA.TheoremQA_post_v2_gen_2c2583 import TheoremQA_datasets
+    from ..humaneval.deprecated_humaneval_gen_d2537e import humaneval_datasets
+    from ..mbpp.deprecated_sanitized_mbpp_gen_cb43ef import sanitized_mbpp_datasets
+
+datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])

opencompass/configs/datasets/collections/base_medium.py

@@ -0,0 +1,56 @@
+from mmengine.config import read_base
+
+with read_base():
+    from ..mmlu.mmlu_ppl_ac766d import mmlu_datasets
+    from ..ceval.ceval_ppl_578f8d import ceval_datasets
+    from ..agieval.agieval_mixed_713d14 import agieval_datasets
+    from ..GaokaoBench.GaokaoBench_mixed_9af5ee import GaokaoBench_datasets
+    from ..bbh.bbh_gen_5b92b0 import bbh_datasets
+    from ..humaneval.humaneval_gen_8e312c import humaneval_datasets
+    from ..mbpp.deprecated_mbpp_gen_1e1056 import mbpp_datasets
+    from ..CLUE_C3.CLUE_C3_ppl_e24a31 import C3_datasets
+    from ..CLUE_CMRC.CLUE_CMRC_gen_1bd3c8 import CMRC_datasets
+    from ..CLUE_DRCD.CLUE_DRCD_gen_1bd3c8 import DRCD_datasets
+    from ..CLUE_afqmc.CLUE_afqmc_ppl_6507d7 import afqmc_datasets
+    from ..CLUE_cmnli.CLUE_cmnli_ppl_fdc6de import cmnli_datasets
+    from ..CLUE_ocnli.CLUE_ocnli_ppl_fdc6de import ocnli_datasets
+    from ..FewCLUE_bustm.FewCLUE_bustm_ppl_e53034 import bustm_datasets
+    from ..FewCLUE_chid.FewCLUE_chid_ppl_8f2872 import chid_datasets
+    from ..FewCLUE_cluewsc.FewCLUE_cluewsc_ppl_4284a0 import cluewsc_datasets
+    from ..FewCLUE_csl.FewCLUE_csl_ppl_841b62 import csl_datasets
+    from ..FewCLUE_eprstmt.FewCLUE_eprstmt_ppl_f1e631 import eprstmt_datasets
+    from ..FewCLUE_ocnli_fc.FewCLUE_ocnli_fc_ppl_c08300 import ocnli_fc_datasets
+    from ..FewCLUE_tnews.FewCLUE_tnews_ppl_d10e8a import tnews_datasets
+    from ..lcsts.lcsts_gen_8ee1fe import lcsts_datasets
+    from ..lambada.lambada_gen_217e11 import lambada_datasets
+    from ..storycloze.storycloze_ppl_496661 import storycloze_datasets
+    from ..SuperGLUE_AX_b.SuperGLUE_AX_b_ppl_6db806 import AX_b_datasets
+    from ..SuperGLUE_AX_g.SuperGLUE_AX_g_ppl_66caf3 import AX_g_datasets
+    from ..SuperGLUE_BoolQ.SuperGLUE_BoolQ_ppl_314b96 import BoolQ_datasets
+    from ..SuperGLUE_CB.SuperGLUE_CB_ppl_0143fe import CB_datasets
+    from ..SuperGLUE_COPA.SuperGLUE_COPA_ppl_9f3618 import COPA_datasets
+    from ..SuperGLUE_MultiRC.SuperGLUE_MultiRC_ppl_ced824 import MultiRC_datasets
+    from ..SuperGLUE_RTE.SuperGLUE_RTE_ppl_66caf3 import RTE_datasets
+    from ..SuperGLUE_ReCoRD.SuperGLUE_ReCoRD_gen_30dea0 import ReCoRD_datasets
+    from ..SuperGLUE_WiC.SuperGLUE_WiC_ppl_312de9 import WiC_datasets
+    from ..SuperGLUE_WSC.SuperGLUE_WSC_ppl_003529 import WSC_datasets
+    from ..race.race_ppl_a138cd import race_datasets
+    from ..Xsum.Xsum_gen_31397e import Xsum_datasets
+    from ..gsm8k.gsm8k_gen_1d7fe4 import gsm8k_datasets
+    from ..summedits.summedits_ppl_1fbeb6 import summedits_datasets
+    from ..math.math_gen_265cce import math_datasets
+    from ..TheoremQA.TheoremQA_gen_ef26ca import TheoremQA_datasets
+    from ..hellaswag.hellaswag_ppl_47bff9 import hellaswag_datasets
+    from ..ARC_e.ARC_e_ppl_a450bd import ARC_e_datasets
+    from ..ARC_c.ARC_c_ppl_a450bd import ARC_c_datasets
+    from ..commonsenseqa.commonsenseqa_ppl_5545e2 import commonsenseqa_datasets
+    from ..piqa.piqa_ppl_1cf9f0 import piqa_datasets
+    from ..siqa.siqa_ppl_ced5f6 import siqa_datasets
+    from ..strategyqa.strategyqa_gen_1180a7 import strategyqa_datasets
+    from ..winogrande.winogrande_ll_c5cf57 import winogrande_datasets
+    from ..obqa.obqa_ppl_c7c154 import obqa_datasets
+    from ..nq.nq_gen_c788f6 import nq_datasets
+    from ..triviaqa.triviaqa_gen_2121ce import triviaqa_datasets
+    from ..flores.flores_gen_806ede import flores_datasets
+
+datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])

opencompass/configs/datasets/collections/base_medium_llama.py

@@ -0,0 +1,56 @@
+from mmengine.config import read_base
+
+with read_base():
+    from ..mmlu.mmlu_ppl_ac766d import mmlu_datasets
+    from ..ceval.ceval_ppl_578f8d import ceval_datasets
+    from ..agieval.agieval_mixed_713d14 import agieval_datasets
+    from ..GaokaoBench.GaokaoBench_mixed_9af5ee import GaokaoBench_datasets
+    from ..bbh.bbh_gen_5b92b0 import bbh_datasets
+    from ..humaneval.deprecated_humaneval_gen_a82cae import humaneval_datasets
+    from ..mbpp.deprecated_mbpp_gen_1e1056 import mbpp_datasets
+    from ..CLUE_C3.CLUE_C3_ppl_e24a31 import C3_datasets
+    from ..CLUE_CMRC.CLUE_CMRC_gen_1bd3c8 import CMRC_datasets
+    from ..CLUE_DRCD.CLUE_DRCD_gen_1bd3c8 import DRCD_datasets
+    from ..CLUE_afqmc.CLUE_afqmc_ppl_6507d7 import afqmc_datasets
+    from ..CLUE_cmnli.CLUE_cmnli_ppl_fdc6de import cmnli_datasets
+    from ..CLUE_ocnli.CLUE_ocnli_ppl_fdc6de import ocnli_datasets
+    from ..FewCLUE_bustm.FewCLUE_bustm_ppl_e53034 import bustm_datasets
+    from ..FewCLUE_chid.FewCLUE_chid_ppl_8f2872 import chid_datasets
+    from ..FewCLUE_cluewsc.FewCLUE_cluewsc_ppl_4284a0 import cluewsc_datasets
+    from ..FewCLUE_csl.FewCLUE_csl_ppl_841b62 import csl_datasets
+    from ..FewCLUE_eprstmt.FewCLUE_eprstmt_ppl_f1e631 import eprstmt_datasets
+    from ..FewCLUE_ocnli_fc.FewCLUE_ocnli_fc_ppl_c08300 import ocnli_fc_datasets
+    from ..FewCLUE_tnews.FewCLUE_tnews_ppl_d10e8a import tnews_datasets
+    from ..lcsts.lcsts_gen_8ee1fe import lcsts_datasets
+    from ..lambada.lambada_gen_217e11 import lambada_datasets
+    from ..storycloze.storycloze_ppl_496661 import storycloze_datasets
+    from ..SuperGLUE_AX_b.SuperGLUE_AX_b_ppl_6db806 import AX_b_datasets
+    from ..SuperGLUE_AX_g.SuperGLUE_AX_g_ppl_66caf3 import AX_g_datasets
+    from ..SuperGLUE_BoolQ.SuperGLUE_BoolQ_ppl_314797 import BoolQ_datasets
+    from ..SuperGLUE_CB.SuperGLUE_CB_ppl_0143fe import CB_datasets
+    from ..SuperGLUE_COPA.SuperGLUE_COPA_ppl_9f3618 import COPA_datasets
+    from ..SuperGLUE_MultiRC.SuperGLUE_MultiRC_ppl_ced824 import MultiRC_datasets
+    from ..SuperGLUE_RTE.SuperGLUE_RTE_ppl_66caf3 import RTE_datasets
+    from ..SuperGLUE_ReCoRD.SuperGLUE_ReCoRD_gen_30dea0 import ReCoRD_datasets
+    from ..SuperGLUE_WiC.SuperGLUE_WiC_ppl_312de9 import WiC_datasets
+    from ..SuperGLUE_WSC.SuperGLUE_WSC_ppl_003529 import WSC_datasets
+    from ..race.race_ppl_5831a0 import race_datasets
+    from ..Xsum.Xsum_gen_31397e import Xsum_datasets
+    from ..gsm8k.gsm8k_gen_1d7fe4 import gsm8k_datasets
+    from ..summedits.summedits_ppl_1fbeb6 import summedits_datasets
+    from ..math.math_gen_265cce import math_datasets
+    from ..TheoremQA.TheoremQA_gen_ef26ca import TheoremQA_datasets
+    from ..hellaswag.hellaswag_ppl_a6e128 import hellaswag_datasets
+    from ..ARC_e.ARC_e_ppl_2ef631 import ARC_e_datasets
+    from ..ARC_c.ARC_c_ppl_2ef631 import ARC_c_datasets
+    from ..commonsenseqa.commonsenseqa_ppl_5545e2 import commonsenseqa_datasets
+    from ..piqa.piqa_ppl_0cfff2 import piqa_datasets
+    from ..siqa.siqa_ppl_e8d8c5 import siqa_datasets
+    from ..strategyqa.strategyqa_gen_1180a7 import strategyqa_datasets
+    from ..winogrande.winogrande_ll_c5cf57 import winogrande_datasets
+    from ..obqa.obqa_ppl_6aac9e import obqa_datasets
+    from ..nq.nq_gen_0356ec import nq_datasets
+    from ..triviaqa.triviaqa_gen_0356ec import triviaqa_datasets
+    from ..flores.flores_gen_806ede import flores_datasets
+
+datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])

opencompass/configs/datasets/collections/base_small.py

@@ -0,0 +1,38 @@
+from mmengine.config import read_base
+
+with read_base():
+    from ..ceval.ceval_ppl_578f8d import ceval_datasets
+    from ..bbh.bbh_gen_5b92b0 import bbh_datasets
+    from ..CLUE_CMRC.CLUE_CMRC_gen_1bd3c8 import CMRC_datasets
+    from ..CLUE_DRCD.CLUE_DRCD_gen_1bd3c8 import DRCD_datasets
+    from ..CLUE_afqmc.CLUE_afqmc_ppl_6507d7 import afqmc_datasets
+    from ..FewCLUE_bustm.FewCLUE_bustm_ppl_e53034 import bustm_datasets
+    from ..FewCLUE_chid.FewCLUE_chid_ppl_8f2872 import chid_datasets
+    from ..FewCLUE_cluewsc.FewCLUE_cluewsc_ppl_868415 import cluewsc_datasets
+    from ..FewCLUE_eprstmt.FewCLUE_eprstmt_ppl_f1e631 import eprstmt_datasets
+    from ..humaneval.humaneval_gen_8e312c import humaneval_datasets
+    from ..mbpp.deprecated_mbpp_gen_1e1056 import mbpp_datasets
+    from ..lambada.lambada_gen_217e11 import lambada_datasets
+    from ..storycloze.storycloze_ppl_496661 import storycloze_datasets
+    from ..SuperGLUE_AX_b.SuperGLUE_AX_b_ppl_6db806 import AX_b_datasets
+    from ..SuperGLUE_AX_g.SuperGLUE_AX_g_ppl_66caf3 import AX_g_datasets
+    from ..SuperGLUE_BoolQ.SuperGLUE_BoolQ_ppl_314b96 import BoolQ_datasets
+    from ..SuperGLUE_CB.SuperGLUE_CB_ppl_0143fe import CB_datasets
+    from ..SuperGLUE_COPA.SuperGLUE_COPA_ppl_9f3618 import COPA_datasets
+    from ..SuperGLUE_MultiRC.SuperGLUE_MultiRC_ppl_ced824 import MultiRC_datasets
+    from ..SuperGLUE_RTE.SuperGLUE_RTE_ppl_66caf3 import RTE_datasets
+    from ..SuperGLUE_ReCoRD.SuperGLUE_ReCoRD_gen_30dea0 import ReCoRD_datasets
+    from ..SuperGLUE_WiC.SuperGLUE_WiC_ppl_312de9 import WiC_datasets
+    from ..SuperGLUE_WSC.SuperGLUE_WSC_ppl_d0f531 import WSC_datasets
+    from ..race.race_ppl_a138cd import race_datasets
+    from ..math.math_gen_265cce import math_datasets
+    from ..gsm8k.gsm8k_gen_1d7fe4 import gsm8k_datasets
+    from ..summedits.summedits_ppl_1fbeb6 import summedits_datasets
+    from ..hellaswag.hellaswag_ppl_47bff9 import hellaswag_datasets
+    from ..piqa.piqa_ppl_1cf9f0 import piqa_datasets
+    from ..winogrande.winogrande_ll_c5cf57 import winogrande_datasets
+    from ..obqa.obqa_ppl_c7c154 import obqa_datasets
+    from ..nq.nq_gen_c788f6 import nq_datasets
+    from ..triviaqa.triviaqa_gen_2121ce import triviaqa_datasets
+
+datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])

opencompass/configs/datasets/collections/chat_core.py

@@ -0,0 +1,20 @@
+from mmengine.config import read_base
+
+with read_base():
+    from ..mmlu.mmlu_gen_4d595a import mmlu_datasets
+    from ..cmmlu.cmmlu_gen_c13365 import cmmlu_datasets
+    from ..ceval.ceval_internal_gen_2daf24 import ceval_datasets
+    from ..GaokaoBench.GaokaoBench_no_subjective_gen_4c31db import GaokaoBench_datasets
+    from ..triviaqa.triviaqa_wiki_1shot_gen_bc5f21 import triviaqa_datasets
+    from ..nq.nq_open_1shot_gen_2e45e5 import nq_datasets
+    from ..race.race_gen_69ee4f import race_datasets
+    from ..winogrande.winogrande_5shot_gen_6447e6 import winogrande_datasets
+    from ..hellaswag.hellaswag_10shot_gen_e42710 import hellaswag_datasets
+    from ..bbh.bbh_gen_5b92b0 import bbh_datasets
+    from ..gsm8k.gsm8k_gen_1d7fe4 import gsm8k_datasets
+    from ..math.math_evaluatorv2_gen_cecb31 import math_datasets
+    from ..TheoremQA.TheoremQA_post_v2_gen_ef26ca import TheoremQA_datasets
+    from ..humaneval.humaneval_gen_8e312c import humaneval_datasets
+    from ..mbpp.deprecated_sanitized_mbpp_gen_1e1056 import sanitized_mbpp_datasets
+
+datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])

opencompass/configs/datasets/collections/chat_medium.py

@@ -0,0 +1,56 @@
+from mmengine.config import read_base
+
+with read_base():
+    from ..mmlu.mmlu_gen_4d595a import mmlu_datasets
+    from ..ceval.ceval_gen_5f30c7 import ceval_datasets
+    from ..agieval.agieval_gen_64afd3 import agieval_datasets
+    from ..GaokaoBench.GaokaoBench_gen_5cfe9e import GaokaoBench_datasets
+    from ..bbh.bbh_gen_5b92b0 import bbh_datasets
+    from ..humaneval.humaneval_gen_8e312c import humaneval_datasets
+    from ..mbpp.deprecated_mbpp_gen_1e1056 import mbpp_datasets
+    from ..CLUE_C3.CLUE_C3_gen_8c358f import C3_datasets
+    from ..CLUE_CMRC.CLUE_CMRC_gen_1bd3c8 import CMRC_datasets
+    from ..CLUE_DRCD.CLUE_DRCD_gen_1bd3c8 import DRCD_datasets
+    from ..CLUE_afqmc.CLUE_afqmc_gen_901306 import afqmc_datasets
+    from ..CLUE_cmnli.CLUE_cmnli_gen_1abf97 import cmnli_datasets
+    from ..CLUE_ocnli.CLUE_ocnli_gen_c4cb6c import ocnli_datasets
+    from ..FewCLUE_bustm.FewCLUE_bustm_gen_634f41 import bustm_datasets
+    from ..FewCLUE_chid.FewCLUE_chid_gen_0a29a2 import chid_datasets
+    from ..FewCLUE_cluewsc.FewCLUE_cluewsc_gen_c68933 import cluewsc_datasets
+    from ..FewCLUE_csl.FewCLUE_csl_gen_28b223 import csl_datasets
+    from ..FewCLUE_eprstmt.FewCLUE_eprstmt_gen_740ea0 import eprstmt_datasets
+    from ..FewCLUE_ocnli_fc.FewCLUE_ocnli_fc_gen_f97a97 import ocnli_fc_datasets
+    from ..FewCLUE_tnews.FewCLUE_tnews_gen_b90e4a import tnews_datasets
+    from ..lcsts.lcsts_gen_8ee1fe import lcsts_datasets
+    from ..lambada.lambada_gen_217e11 import lambada_datasets
+    from ..storycloze.storycloze_gen_7f656a import storycloze_datasets
+    from ..SuperGLUE_AX_b.SuperGLUE_AX_b_gen_4dfefa import AX_b_datasets
+    from ..SuperGLUE_AX_g.SuperGLUE_AX_g_gen_68aac7 import AX_g_datasets
+    from ..SuperGLUE_BoolQ.SuperGLUE_BoolQ_gen_883d50 import BoolQ_datasets
+    from ..SuperGLUE_CB.SuperGLUE_CB_gen_854c6c import CB_datasets
+    from ..SuperGLUE_COPA.SuperGLUE_COPA_gen_91ca53 import COPA_datasets
+    from ..SuperGLUE_MultiRC.SuperGLUE_MultiRC_gen_27071f import MultiRC_datasets
+    from ..SuperGLUE_RTE.SuperGLUE_RTE_gen_68aac7 import RTE_datasets
+    from ..SuperGLUE_ReCoRD.SuperGLUE_ReCoRD_gen_30dea0 import ReCoRD_datasets
+    from ..SuperGLUE_WiC.SuperGLUE_WiC_gen_d06864 import WiC_datasets
+    from ..SuperGLUE_WSC.SuperGLUE_WSC_gen_7902a7 import WSC_datasets
+    from ..race.race_gen_69ee4f import race_datasets
+    from ..Xsum.Xsum_gen_31397e import Xsum_datasets
+    from ..gsm8k.gsm8k_gen_1d7fe4 import gsm8k_datasets
+    from ..summedits.summedits_gen_315438 import summedits_datasets
+    from ..math.math_gen_265cce import math_datasets
+    from ..TheoremQA.TheoremQA_gen_7009de import TheoremQA_datasets
+    from ..hellaswag.hellaswag_gen_6faab5 import hellaswag_datasets
+    from ..ARC_e.ARC_e_gen_1e0de5 import ARC_e_datasets
+    from ..ARC_c.ARC_c_gen_1e0de5 import ARC_c_datasets
+    from ..commonsenseqa.commonsenseqa_gen_c946f2 import commonsenseqa_datasets
+    from ..piqa.piqa_gen_1194eb import piqa_datasets
+    from ..siqa.siqa_gen_e78df3 import siqa_datasets
+    from ..strategyqa.strategyqa_gen_1180a7 import strategyqa_datasets
+    from ..winogrande.deprecated_winogrande_gen_a9ede5 import winogrande_datasets
+    from ..obqa.obqa_gen_9069e4 import obqa_datasets
+    from ..nq.nq_gen_c788f6 import nq_datasets
+    from ..triviaqa.triviaqa_gen_2121ce import triviaqa_datasets
+    from ..flores.flores_gen_806ede import flores_datasets
+
+datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])

opencompass/configs/datasets/collections/chat_small.py

@@ -0,0 +1,39 @@
+from mmengine.config import read_base
+
+with read_base():
+    from ..mmlu.mmlu_gen_4d595a import mmlu_datasets
+    from ..ceval.ceval_gen_5f30c7 import ceval_datasets
+    from ..bbh.bbh_gen_5b92b0 import bbh_datasets
+    from ..CLUE_CMRC.CLUE_CMRC_gen_1bd3c8 import CMRC_datasets
+    from ..CLUE_DRCD.CLUE_DRCD_gen_1bd3c8 import DRCD_datasets
+    from ..CLUE_afqmc.CLUE_afqmc_gen_901306 import afqmc_datasets
+    from ..FewCLUE_bustm.FewCLUE_bustm_gen_634f41 import bustm_datasets
+    from ..FewCLUE_chid.FewCLUE_chid_gen_0a29a2 import chid_datasets
+    from ..FewCLUE_cluewsc.FewCLUE_cluewsc_gen_c68933 import cluewsc_datasets
+    from ..FewCLUE_eprstmt.FewCLUE_eprstmt_gen_740ea0 import eprstmt_datasets
+    from ..humaneval.humaneval_gen_8e312c import humaneval_datasets
+    from ..mbpp.deprecated_mbpp_gen_1e1056 import mbpp_datasets
+    from ..lambada.lambada_gen_217e11 import lambada_datasets
+    from ..storycloze.storycloze_gen_7f656a import storycloze_datasets
+    from ..SuperGLUE_AX_b.SuperGLUE_AX_b_gen_4dfefa import AX_b_datasets
+    from ..SuperGLUE_AX_g.SuperGLUE_AX_g_gen_68aac7 import AX_g_datasets
+    from ..SuperGLUE_BoolQ.SuperGLUE_BoolQ_gen_883d50 import BoolQ_datasets
+    from ..SuperGLUE_CB.SuperGLUE_CB_gen_854c6c import CB_datasets
+    from ..SuperGLUE_COPA.SuperGLUE_COPA_gen_91ca53 import COPA_datasets
+    from ..SuperGLUE_MultiRC.SuperGLUE_MultiRC_gen_27071f import MultiRC_datasets
+    from ..SuperGLUE_RTE.SuperGLUE_RTE_gen_68aac7 import RTE_datasets
+    from ..SuperGLUE_ReCoRD.SuperGLUE_ReCoRD_gen_30dea0 import ReCoRD_datasets
+    from ..SuperGLUE_WiC.SuperGLUE_WiC_gen_d06864 import WiC_datasets
+    from ..SuperGLUE_WSC.SuperGLUE_WSC_gen_fe4bf3 import WSC_datasets
+    from ..race.race_gen_69ee4f import race_datasets
+    from ..math.math_gen_265cce import math_datasets
+    from ..gsm8k.gsm8k_gen_1d7fe4 import gsm8k_datasets
+    from ..summedits.summedits_gen_315438 import summedits_datasets
+    from ..hellaswag.hellaswag_gen_6faab5 import hellaswag_datasets
+    from ..piqa.piqa_gen_1194eb import piqa_datasets
+    from ..winogrande.deprecated_winogrande_gen_a9ede5 import winogrande_datasets
+    from ..obqa.obqa_gen_9069e4 import obqa_datasets
+    from ..nq.nq_gen_c788f6 import nq_datasets
+    from ..triviaqa.triviaqa_gen_2121ce import triviaqa_datasets
+
+datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])

opencompass/configs/datasets/collections/example.py

@@ -0,0 +1,7 @@
+from mmengine.config import read_base
+
+with read_base():
+    from ..piqa.piqa_gen_1194eb import piqa_datasets
+    from ..nq.nq_gen_c788f6 import nq_datasets
+
+datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])

opencompass/configs/datasets/ds1000/ds1000_compl_gen_cbc84f.py

@@ -0,0 +1,69 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import (DS1000Dataset, ds1000_completion_postprocess,
+                                  ds1000_matplotlib_postprocess,
+                                  DS1000Evaluator)
+
+ds1000_reader_cfg = dict(
+    input_columns=['prompt'],
+    output_column='test_column',
+    train_split='test',
+    test_split='test')
+
+ds1000_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='{prompt}',
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+ds1000_eval_cfg = dict(
+    evaluator=dict(type=DS1000Evaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=ds1000_completion_postprocess),
+)
+
+# The DS-1000 dataset can be downloaded from
+# https://github.com/HKUNLP/DS-1000/blob/main/ds1000_data.zip
+ds1000_datasets = [
+    dict(
+        abbr=f'ds1000_{lib}',
+        type=DS1000Dataset,
+        path='./data/ds1000_data/',
+        libs=f'{lib}',
+        mode='Completion',
+        reader_cfg=ds1000_reader_cfg,
+        infer_cfg=ds1000_infer_cfg,
+        eval_cfg=ds1000_eval_cfg,
+    ) for lib in [
+        'Pandas',
+        'Numpy',
+        'Tensorflow',
+        'Scipy',
+        'Sklearn',
+        'Pytorch',
+    ]
+]
+ds1000_datasets.append(
+    dict(
+        abbr='ds1000_Matplotlib',
+        type=DS1000Dataset,
+        path='./data/ds1000_data/',
+        libs='Matplotlib',
+        mode='Completion',
+        reader_cfg=ds1000_reader_cfg,
+        infer_cfg=ds1000_infer_cfg,
+        eval_cfg=dict(
+            evaluator=dict(type=DS1000Evaluator),
+            pred_role='BOT',
+            pred_postprocessor=dict(type=ds1000_matplotlib_postprocess),
+        ),
+    ))

opencompass/configs/datasets/ds1000/ds1000_compl_service_eval_gen_cbc84f.py

@@ -0,0 +1,68 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import DS1000Dataset, DS1000ServiceEvaluator
+
+ds1000_reader_cfg = dict(
+    input_columns=['prompt'],
+    output_column='test_column',
+    train_split='test',
+    test_split='test')
+
+ds1000_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='{prompt}',
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+ds1000_eval_cfg_dict = {
+    lib: dict(
+        evaluator=dict(
+            type=DS1000ServiceEvaluator,
+            lib=lib,
+            ip_address=
+            'localhost',  # replace to your code_eval_server ip_address, port
+            port=5000
+            ),
+        pred_role='BOT')
+    for lib in [
+        'Pandas',
+        'Numpy',
+        'Tensorflow',
+        'Scipy',
+        'Sklearn',
+        'Pytorch',
+        'Matplotlib',
+    ]
+}
+
+# The DS-1000 dataset can be downloaded from
+# https://github.com/HKUNLP/DS-1000/blob/main/ds1000_data.zip
+ds1000_datasets = [
+    dict(
+        abbr=f'ds1000_{lib}',
+        type=DS1000Dataset,
+        path='./data/ds1000_data/',
+        libs=f'{lib}',
+        mode='Completion',
+        reader_cfg=ds1000_reader_cfg,
+        infer_cfg=ds1000_infer_cfg,
+        eval_cfg=ds1000_eval_cfg_dict[lib],
+    ) for lib in [
+        'Pandas',
+        'Numpy',
+        'Tensorflow',
+        'Scipy',
+        'Sklearn',
+        'Pytorch',
+        'Matplotlib',
+    ]
+]

opencompass/configs/datasets/ds1000/ds1000_gen_5c4bec.py

@@ -0,0 +1,84 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import AgentInferencer
+from opencompass.datasets import DS1000Dataset_Interperter, DS1000InterpreterEvaluator
+
+ds1000_example = """
+In the following task, you should generate code with one assertion to testify the correctness of your code.
+
+Example:
+
+<HUMAN>Problem:
+How do I get the dimensions of an array? For instance, this is (2, 2):
+a = np.array([[1,2],[3,4]])
+<ASSISTANT>{thought} In Python, Numpy provides a method called `shape` which helps to get the dimensions of an array.
+{action} PythonInterpreter
+{action_input}
+```python
+import numpy as np
+def solution(x):
+    # Convert to np.ndarray
+    x = np.array(x)
+    # Getting the dimensions of the array
+    dimensions = x.shape
+    return dimensions
+assert solution([[1,2],[3,4]]) == (2, 2)
+```
+<SYSTEM>{response}True
+<ASSISTANT> {thought} By running this code, you can get the dimensions of an array.
+{finish}
+```python
+import numpy as np
+def solution(x):
+    # Convert to np.ndarray
+    x = np.array(x)
+    # Getting the dimensions of the array
+    dimensions = x.shape
+    return dimensions
+```
+"""
+
+ds1000_reader_cfg = dict(
+    input_columns=['prompt'],
+    output_column='test_column',
+    train_split='test',
+    test_split='test',
+)
+
+ds1000_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template="""{prompt}""",
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=AgentInferencer, example=ds1000_example),
+)
+
+ds1000_eval_cfg = dict(
+    evaluator=dict(type=DS1000InterpreterEvaluator),
+    pred_role='BOT',
+)
+
+# The DS-1000 dataset can be downloaded from
+# https://github.com/HKUNLP/DS-1000/blob/main/ds1000_data.zip
+
+# Matplotlib cannot fit this setting
+ds1000_datasets = [
+    dict(
+        abbr=f'ds1000_{lib}',
+        type=DS1000Dataset_Interperter,  # bustm share the same format with AFQMC
+        path='./data/ds1000_data/',
+        libs=f'{lib}',
+        reader_cfg=ds1000_reader_cfg,
+        infer_cfg=ds1000_infer_cfg,
+        eval_cfg=ds1000_eval_cfg,
+    )
+    for lib in [
+        'Pandas',
+        'Numpy',
+        # 'Tensorflow',  # error using tensorflow, skipped temporarily
+        'Scipy',
+        'Sklearn',
+        'Pytorch',
+    ]
+]

opencompass/configs/datasets/ds1000/ds1000_gen_cbc84f.py

@@ -0,0 +1,67 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import (DS1000Dataset, ds1000_postprocess,
+                                  ds1000_matplotlib_postprocess,
+                                  DS1000Evaluator)
+
+ds1000_reader_cfg = dict(
+    input_columns=['prompt'],
+    output_column='test_column',
+    train_split='test',
+    test_split='test')
+
+ds1000_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='{prompt}',
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+ds1000_eval_cfg = dict(
+    evaluator=dict(type=DS1000Evaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=ds1000_postprocess),
+)
+
+# The DS-1000 dataset can be downloaded from
+# https://github.com/HKUNLP/DS-1000/blob/main/ds1000_data.zip
+ds1000_datasets = [
+    dict(
+        abbr=f'ds1000_{lib}',
+        type=DS1000Dataset,
+        path='./data/ds1000_data/',
+        libs=f'{lib}',
+        reader_cfg=ds1000_reader_cfg,
+        infer_cfg=ds1000_infer_cfg,
+        eval_cfg=ds1000_eval_cfg,
+    ) for lib in [
+        'Pandas',
+        'Numpy',
+        'Tensorflow',
+        'Scipy',
+        'Sklearn',
+        'Pytorch',
+    ]
+]
+ds1000_datasets.append(
+    dict(
+        abbr='ds1000_Matplotlib',
+        type=DS1000Dataset,
+        path='./data/ds1000_data/',
+        libs='Matplotlib',
+        reader_cfg=ds1000_reader_cfg,
+        infer_cfg=ds1000_infer_cfg,
+        eval_cfg=dict(
+            evaluator=dict(type=DS1000Evaluator),
+            pred_role='BOT',
+            pred_postprocessor=dict(type=ds1000_matplotlib_postprocess),
+        ),
+    ))

opencompass/configs/datasets/ds1000/ds1000_service_eval_gen_cbc84f.py

@@ -0,0 +1,67 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import DS1000Dataset, DS1000ServiceEvaluator
+
+ds1000_reader_cfg = dict(
+    input_columns=['prompt'],
+    output_column='test_column',
+    train_split='test',
+    test_split='test')
+
+ds1000_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='{prompt}',
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+ds1000_eval_cfg_dict = {
+    lib: dict(
+        evaluator=dict(
+            type=DS1000ServiceEvaluator,
+            lib=lib,
+            ip_address=
+            'localhost',  # replace to your code_eval_server ip_address, port
+            port=5000
+            ),
+        pred_role='BOT')
+    for lib in [
+        'Pandas',
+        'Numpy',
+        'Tensorflow',
+        'Scipy',
+        'Sklearn',
+        'Pytorch',
+        'Matplotlib',
+    ]
+}
+
+# The DS-1000 dataset can be downloaded from
+# https://github.com/HKUNLP/DS-1000/blob/main/ds1000_data.zip
+ds1000_datasets = [
+    dict(
+        abbr=f'ds1000_{lib}',
+        type=DS1000Dataset,
+        path='./data/ds1000_data/',
+        libs=f'{lib}',
+        reader_cfg=ds1000_reader_cfg,
+        infer_cfg=ds1000_infer_cfg,
+        eval_cfg=ds1000_eval_cfg_dict[lib],
+    ) for lib in [
+        'Pandas',
+        'Numpy',
+        'Tensorflow',
+        'Scipy',
+        'Sklearn',
+        'Pytorch',
+        'Matplotlib',
+    ]
+]

opencompass/configs/datasets/gaokao_math/README.md

@@ -0,0 +1,108 @@
+# GaoKao MATH Answer Evaluation Dataset
+A dataset for testing the performance of the model in the GaoKao MATH Answer Extraction task.
+Now support the following format of GAOKAO math questions:
+1. '单选题'：Single choice question
+2. '多选题'：Multiple choice question
+3. '填空题'：Fill in the blank question, can be multiple blanks
+4. '解答题'：Answer question, can be multiple answers
+
+Sample data:
+```json
+[
+    {
+        "id": "3b270bc4-570a-4d77-b122-a2fc372f7d6a",
+        "question": "过椭圆${x^2\\over {16}} +{ y^2 \\over {4}}=1$ %内一点$M(2,1)$ %引一条弦，使该弦被点$M$ %平分，则这条弦所在直线的方程为（ ）．\nA. $x+2y-4=0$ %\nB. $x-2y-4=0$ %\nC. $x+2y+4=0$ %\nD. $x-2y+4=0$ %\n\n",
+        "response": "本题主要考查直线与圆锥曲线．设所求直线与椭圆的一个交点为$A(x,y)$ %，由于中点$M(2,1)$ %，所以另一个交点$B$ %为$(4-x,2-y)$ %．因为$A$ %，$B$ %两点都在椭圆上，所以$x^2+4y^2=16$ %，$(4-x)^2+4(2-y)^2=16$ %，两式相减，整理可得$x+2y-4=0$ %．由于过$A$ %，$B$ %两点的直线只有一条，所以这条弦所在直线的方程为$x+2y-4=0$ %．故本题正确答案为A．\n答案是：A",
+        "extract_answer": "A",
+        "question_type": "单选题"
+    },
+    {
+        "id": "d60e42d7-30ee-44f9-a94d-aff6a8127750",
+        "question": "若函数$f(x)$ 具有下列性质：1.定义域为$(-1,1)$ ；2.对于任意的$x,y\\in(-1,1)$ ，都有$f(x)+f(y)=f\\left({\\dfrac{x+y}{1+xy}}\\right)$ ；3.当$-1< x< 0$ 时，$f(x)>0$ ，则称函数$f(x)$ 为$δ$ 的函数$.$ 若函数$f(x)$ 为$δ$ 的函数，则以下结论正确的是$(\\quad)$\nA. $\nB. x)$ 为奇函数\nC. $\nD. x)$ 为偶函数\nE. $\nF. x)$ 为单调递减函数\nG. $\nH. x)$ 为单调递增函数\n\n",
+        "response": "函数$f(x)$ 为$δ$ 的函数，令$x=y=0$ ，则$f(0)+f(0)=f(0)$ ，即$f(0)=0$ ，令$y=-x$ ，则$f(x)+f(-x)=f\\left(\\dfrac{x-x}{1-{x}^{2}}\\right)=f(0)=0$ ，则$f(-x)=-f(x)$ ，即函数$f(x)$ 是奇函数，设$-1< x< y< 1$ ，则$f(x)-f(y)=f(x)+f(-y)=f\\left(\\dfrac{x-y}{1-xy}\\right)$ ，$∵-1< x< y< 1$ ，$∴-1< \\dfrac{x-y}{1-xy}< 0$ ，则$f\\left(\\dfrac{x-y}{1-xy}\\right)>0$ ，即$f(x)-f(y)>0$ ，则$f(x)>f(y)$ ，即$f(x)$ 在$(-1,1)$ 上是减函数.故选$AC.$ 本题考查函数的奇偶性和单调性的判断，注意运用定义法，考查运算能力和推理能力，属于中档题.可令$x=y=0$ ，求得$f(0)=0$ ，再令$y=-x$ 可得$f(-x)=-f(x)$ ，可得$f(x)$ 的奇偶性；再令$-1< x< y< 1$ ，运用单调性的定义，结合其偶性的定义可得其单调性．\n答案是：A; C",
+        "extract_answer": "A, C",
+        "question_type": "多选题"
+    },
+    {
+        "id": "31b3f702-e60c-4a20-9a40-73bd72b92d1e",
+        "question": "请完成以下题目(1)曲线$$y=-5\\text{e}^{x}+3$$在点$$(0,-2)$$处的切线方程为___.(2)若曲线$$f(x)=x \\sin x+1$$在$$x=\\dfrac{ \\pi }{2}$$处的切线与直线$$ax+2y+1=0$$相互垂直,则实数$$a=$$___.\n\n",
+        "response": "(1)由$$y=-5\\text{e}^{x}+3$$,得$$y'=-5\\text{e}^{x}$$,所以切线的斜率$$k=y'|_{x=0}=-5$$,所以切线方程为$$y+2=-5(x-0)$$,即$$5x+y+2=0$$.(2)因为$$f'(x)= \\sin x+x \\cos x$$,所以$$f'\\left(\\dfrac{ \\pi }{2}\\right)= \\sin \\dfrac{ \\pi }{2}+\\dfrac{ \\pi }{2}\\cdot \\cos \\dfrac{ \\pi }{2}=1$$.又直线$$ax+2y+1=0$$的斜率为$$-\\dfrac{a}{2}$$,所以根据题意得$$1\\times \\left(-\\dfrac{a}{2}\\right)=-1$$,解得$$a=2$$.\n答案是：(1)$$5x+y+2=0$$ (2)$$2$$",
+        "extract_answer": "['(1)$$5x+y+2=0$$ (2)$$2$$']",
+        "question_type": "填空题"
+    },
+    {
+        "id": "16878941-1772-4290-bc61-00b193d5cf70",
+        "question": "已知函数$f\\left( x \\right)=\\left| 2x-1 \\right|$.（1）若不等式$f\\left( x+\\frac{1}{2} \\right)\\ge 2m+1\\left( m > 0 \\right)$的解集为$\\left( -\\infty ,-2 \\right]\\bigcup \\left[ 2,+\\infty \\right)$，求实数$m$的值；（2）若不等式$f\\left( x \\right)\\le {{2}^{y}}+\\frac{a}{{{2}^{y}}}+\\left| 2x+3 \\right|$对任意的实数$x,y\\in R$恒成立，求实数$a$的最小值.\n\n",
+        "response": "（1）直接写出不等式，解含有绝对值的函数不等式即可；（2）这是恒成立求参的问题,根据绝对值三角不等式得到左侧函数的最值，再结合均值不等式得最值.（1）由条件得$\\left| 2x \\right|\\le 2m+1$得$-m-\\frac{1}{2}\\le x\\le m+\\frac{1}{2}$，所以$m=\\frac{3}{2}$.（2）原不等式等价于$\\left| 2x-1 \\right|-\\left| 2x+3 \\right|\\le {{2}^{y}}+\\frac{a}{{{2}^{y}}}$，而$\\left| 2x-1 \\right|-\\left| 2x+3 \\right|\\le \\left| \\left( 2x-1 \\right)-\\left( 2x+3 \\right) \\right|=4$，所以${{2}^{y}}+\\frac{a}{{{2}^{y}}}\\ge 4$，则$a\\ge {{\\left[ {{2}^{y}}\\left( 4-{{2}^{y}} \\right) \\right]}_{\\text{max}}}=4$，当且仅当$y=1$时取得.\n答案是：(1) $m=\\frac{3}{2}$；(2) 最小值为$a=4$.",
+        "extract_answer": [
+            "(1) $m=\\frac{3}{2}$；(2) 最小值为$a=4$."
+        ],
+        "question_type": "解答题"
+    }
+]
+```
+## How to use
+
+### 1. Prepare the dataset
+```bash
+cd opencompass
+cp -rf /cpfs01/shared/public/liuhongwei/data/gaokao_math_dataset/gaokao_math ./data
+```
+📢：If you want to evaluate your own gaokao math data, replace the `test_v2.jsonl` with your own data, but follow the format above.
+
+### 2. Set the evaluation model
+
+open `opencompass.datasets.gaokao_math.gaokao_math_gen_9b076f` and set the model name and api url for evaluation, multiple urls are supported for acceleration.
+
+```python
+...
+
+gaokao_math_eval_cfg = dict(
+    evaluator=dict(type=GaoKaoMATHEvaluator, model_name='EVALUATE_MODEL_NAME', url=['http://0.0.0.0:23333/v1', 'http://...']))
+
+...
+
+```
+We recommand `Qwen2.5-72B-Instruct` model for evaluation.
+
+
+### 3. Set Extractor model and run the evaluation
+
+```python
+from mmengine.config import read_base
+from opencompass.models import HuggingFacewithChatTemplate
+
+
+with read_base():
+    from opencompass.datasets.gaokao_math.gaokao_math_gen_9b076f import gaokao_math_datasets
+
+
+trained_qwen2_1_5b_model = [ # trained extractor model
+    dict(
+        type=HuggingFacewithChatTemplate,
+        abbr='gaokao_math_extractor_1_5b_v02',
+        path='/cpfs01/shared/public/liuhongwei/models/gaokao_math_trained/gaokao_math_extractor_1_5b_v02',
+        max_out_len=1024,
+        batch_size=8,
+        run_cfg=dict(num_gpus=1),
+    )
+]
+
+datasets = sum([v for k, v in locals().items() if k.endswith("_datasets")], [])
+models = sum([v for k, v in locals().items() if k.endswith("_model")], [])
+
+...
+```
+
+### 4. Run the evaluation
+
+```bash
+python run.py eval.py --dump-eval-details # eval and dump the evaluation details to `results` folder
+```
+
+
+### 5. Evaluation results
+
+| Evaluator / Extractor | Qwen2.5-72B-Instruct | gaokao_math_extractor_1.5b_v0.2 |
+|-----------------------|-----------------------|----------------------------------|
+| Qwen2.5-72B-Instruct (ACC) | 95.85 | 95.2 |

opencompass/configs/datasets/gaokao_math/gaokao_math_gen_f5fd28.py

@@ -0,0 +1,48 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import GaoKaoMATHDataset, GaoKaoMATHEvaluator
+
+
+MATH_CN_PROMPT="""
+你是一个数学阅卷专家，任务是从给定的回答句子中提取精确的关键答案。你必须只提供提取的关键答案，不包括任何额外的文字。
+—
+我将为你提供一个问题、回答句子和问题类型。回答句子是对所提供问题的回应。利用提供的信息，你必须准确而精确地确定并从回答句子中提取预期的关键答案。请不要对问题发表主观看法。
+
+对于单选题，答案应该是选项字母，例如 "A"；
+对于多选题，答案应该是一个选项字母的列表，例如 ["A"] 或 ["A", "B", "C"]；
+对于填空题，答案应该是一个填入空白处的答案列表，列表的数量应该与问题中的空白数量相同，例如 ["$$\\frac{{1}}{{2}}$$"] 或 ["$$\\frac{{1}}{{2}}$$", "2"]。
+对于问答题，类似填空题，为每个小问抽出相应答案，例如 ["$$\\frac{{1}}{{2}}$$"] 或 ["$$\\frac{{1}}{{2}}$$", "2"]。
+
+如果回答句子提供了多个不同的答案，请仔细判断后面提供的答案是否是对前面答案的修正或修改。如果是这样，提取这个修正或修改后的答案作为最终答案。相反，如果回答句子在多个答案之间波动而没有明确的最终答案，你应该输出 [No valid answer]。
+—
+问题类型: {question_type}
+原始问题: {question}
+回答: {response}
+提取的关键答案:
+"""
+
+gaokao_math_reader_cfg = dict(input_columns=['question', 'response', 'question_type'], output_column='extract_answer')
+
+
+gaokao_math_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(role='HUMAN', prompt=MATH_CN_PROMPT),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512))
+
+gaokao_math_eval_cfg = dict(
+    evaluator=dict(type=GaoKaoMATHEvaluator, model_name='Qwen/Qwen2.5-72B-Instruct', url=['http://22.8.73.119:23333/v1', 'http://22.8.4.97:23333/v1', 'http://22.8.22.254:23333/v1', 'http://22.8.17.14:23333/v1']))
+
+gaokao_math_datasets = [
+    dict(
+        type=GaoKaoMATHDataset,
+        abbr='GaoKaoMATH',
+        path='./data/gaokao_math/test_2k.json',
+        reader_cfg=gaokao_math_reader_cfg,
+        infer_cfg=gaokao_math_infer_cfg,
+        eval_cfg=gaokao_math_eval_cfg)
+]

opencompass/configs/datasets/humaneval_cn/humaneval_cn_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .humaneval_cn_gen_6313aa import humaneval_cn_datasets  # noqa: F401, F403

opencompass/configs/datasets/humaneval_cn/humaneval_cn_gen_6313aa.py

@@ -0,0 +1,37 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2
+
+humaneval_reader_cfg = dict(
+    input_columns=['prompt'], output_column='task_id', train_split='test')
+
+# TODO: allow empty output-column
+humaneval_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='完成以下Python代码任务:\n{prompt}'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512))
+
+humaneval_eval_cfg = dict(
+    evaluator=dict(type=HumanEvalEvaluator),
+    pred_role='BOT',
+    k=[1, 10, 100],  # the parameter only for humaneval
+    pred_postprocessor=dict(type=humaneval_postprocess_v2),
+)
+
+humaneval_cn_datasets = [
+    dict(
+        abbr='openai_humaneval_cn',
+        type=HumanevalDataset,
+        path='opencompass/humaneval_cn',
+        reader_cfg=humaneval_reader_cfg,
+        infer_cfg=humaneval_infer_cfg,
+        eval_cfg=humaneval_eval_cfg)
+
+]

opencompass/configs/datasets/humaneval_cn/humaneval_cn_passk_gen_6313aa.py

@@ -0,0 +1,37 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2
+
+humaneval_reader_cfg = dict(
+    input_columns=['prompt'], output_column='task_id', train_split='test')
+
+# TODO: allow empty output-column
+humaneval_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='完成以下Python代码任务:\n{prompt}'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512))
+
+humaneval_eval_cfg = dict(
+    evaluator=dict(type=HumanEvalEvaluator),
+    pred_role='BOT',
+    k=[1, 10, 100],  # the parameter only for humaneval
+    pred_postprocessor=dict(type=humaneval_postprocess_v2),
+)
+
+humaneval_cn_datasets = [
+    dict(
+        abbr='openai_humaneval_cn_passk',
+        type=HumanevalDataset,
+        path='opencompass/humaneval_cn',
+        reader_cfg=humaneval_reader_cfg,
+        infer_cfg=humaneval_infer_cfg,
+        eval_cfg=humaneval_eval_cfg)
+
+]

opencompass/configs/datasets/humaneval_cn/humaneval_cn_repeat10_gen_6313aa.py

@@ -0,0 +1,38 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2
+
+humaneval_reader_cfg = dict(
+    input_columns=['prompt'], output_column='task_id', train_split='test')
+
+# TODO: allow empty output-column
+humaneval_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='完成以下Python代码任务:\n{prompt}'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512))
+
+humaneval_eval_cfg = dict(
+    evaluator=dict(type=HumanEvalEvaluator),
+    pred_role='BOT',
+    k=[1, 10, 100],  # the parameter only for humaneval
+    pred_postprocessor=dict(type=humaneval_postprocess_v2),
+)
+
+humaneval_cn_datasets = [
+    dict(
+        abbr='openai_humaneval_cn_repeat10',
+        type=HumanevalDataset,
+        path='opencompass/humaneval_cn',
+        num_repeats=10,
+        reader_cfg=humaneval_reader_cfg,
+        infer_cfg=humaneval_infer_cfg,
+        eval_cfg=humaneval_eval_cfg)
+
+]

opencompass/configs/datasets/jigsawmultilingual/jigsawmultilingual_clp.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .jigsawmultilingual_clp_fe50d8 import jigsawmultilingual_datasets  # noqa: F401, F403

opencompass/configs/datasets/jigsawmultilingual/jigsawmultilingual_clp_1af0ae.py

@@ -0,0 +1,43 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import CLPInferencer
+from opencompass.openicl.icl_evaluator import AUCROCEvaluator
+from opencompass.datasets import JigsawMultilingualDataset
+
+jigsawmultilingual_reader_cfg = dict(
+    input_columns=['text'],
+    output_column='label',
+    train_split='test',
+    test_split='test')
+
+# Jigsaw multilingual toxic dataset is not supported in Huggingface
+# Please download the test dataset files `test.csv` and `test_labels.csv` from
+# below link and move them the data/ folder to perform evaluation.
+# https://www.kaggle.com/competitions/jigsaw-multilingual-toxic-comment-classification/data
+jigsawmultilingual_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template='Text: {text}\nQuestion: '
+        'Does the above text contain rude, hateful, aggressive, disrespectful '
+        'or unreasonable language?\nAnswer:'),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=CLPInferencer))
+
+jigsawmultilingual_eval_cfg = dict(evaluator=dict(type=AUCROCEvaluator), )
+
+lang = ['es', 'fr', 'it', 'pt', 'ru', 'tr']
+jigsawmultilingual_datasets = []
+
+for _l in lang:
+    jigsawmultilingual_datasets.append(
+        dict(
+            abbr=f'jigsaw_multilingual_{_l}',
+            type=JigsawMultilingualDataset,
+            path='data/jigsawmultilingual/test.csv',
+            label='data/jigsawmultilingual/test_labels.csv',
+            lang=_l,
+            reader_cfg=jigsawmultilingual_reader_cfg,
+            infer_cfg=jigsawmultilingual_infer_cfg,
+            eval_cfg=jigsawmultilingual_eval_cfg))
+
+del lang, _l

opencompass/configs/datasets/jigsawmultilingual/jigsawmultilingual_clp_fe50d8.py

@@ -0,0 +1,47 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import CLPInferencer
+from opencompass.openicl.icl_evaluator import AUCROCEvaluator
+from opencompass.datasets import JigsawMultilingualDataset
+
+jigsawmultilingual_reader_cfg = dict(
+    input_columns=['text'],
+    output_column='label',
+    train_split='test',
+    test_split='test')
+
+# Jigsaw multilingual toxic dataset is not supported in Huggingface
+# Please download the test dataset files `test.csv` and `test_labels.csv` from
+# below link and move them the data/ folder to perform evaluation.
+# https://www.kaggle.com/competitions/jigsaw-multilingual-toxic-comment-classification/data
+jigsawmultilingual_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='Text: {text}\nQuestion: Does the above text contain '
+                'rude, hateful, aggressive, disrespectful or unreasonable '
+                'language?\nAnswer:')
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=CLPInferencer))
+
+jigsawmultilingual_eval_cfg = dict(evaluator=dict(type=AUCROCEvaluator), )
+
+lang = ['es', 'fr', 'it', 'pt', 'ru', 'tr']
+jigsawmultilingual_datasets = []
+
+for _l in lang:
+    jigsawmultilingual_datasets.append(
+        dict(
+            abbr=f'jigsaw_multilingual_{_l}',
+            type=JigsawMultilingualDataset,
+            path='data/jigsawmultilingual/test.csv',
+            label='data/jigsawmultilingual/test_labels.csv',
+            lang=_l,
+            reader_cfg=jigsawmultilingual_reader_cfg,
+            infer_cfg=jigsawmultilingual_infer_cfg,
+            eval_cfg=jigsawmultilingual_eval_cfg))
+
+del lang, _l

opencompass/configs/datasets/livecodebench/README.md

@@ -0,0 +1,42 @@
+# LiveCodeBench
+
+## Dataset
+
+LiveCodeBench provides holistic and contamination-free evaluation of coding capabilities of LLMs. Particularly, LiveCodeBench continuously collects new problems over time from contests across three competition platforms -- LeetCode, AtCoder, and CodeForces. Next, LiveCodeBench also focuses on a broader range of code-related capabilities, such as self-repair, code execution, and test output prediction, beyond just code generation. Currently, LiveCodeBench hosts four hundred high-quality coding problems that were published between May 2023 and March 2024.
+
+- Origin Project: https://livecodebench.github.io/leaderboard.html
+
+## Setting
+
+| Model Type | Code Generation | Test Output Prediction | Code Execution |
+|------------|--------|--------|--------|
+| Base Model    |  ❌      | ❌          |  ❌         |
+| Chat Model    | ✅        |    ✅      |  ✅       |
+
+
+
+## Baseline Performance
+
+
+| Model Type | Code Generation(pass@1) | Test Output Prediction(pass@1) | Code Execution(pass@1) |
+|------------|--------|--------|--------|
+|  Qwen2.5-7B-Instruct(HF)    |  39.25      | 48.64          | 41.96         |
+|  Meta-Llama-3.1-8B-Instruct(HF)  | 20.25       |   24.66     |  17.12      |
+
+
+## Citation
+
+```bibtex
+@article{jain2024livecodebench,
+  author    = {Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica},
+  title     = {LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code},
+  year      = {2024},
+  journal   = {arXiv preprint},
+}
+@misc{2023opencompass,
+    title={OpenCompass: A Universal Evaluation Platform for Foundation Models},
+    author={OpenCompass Contributors},
+    howpublished = {\url{https://github.com/open-compass/opencompass}},
+    year={2023}
+}
+```

opencompass/configs/datasets/livecodebench/livecodebench_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .livecodebench_gen_6966bc import LCB_datasets  # noqa: F401, F403

opencompass/configs/datasets/livecodebench/livecodebench_gen_6966bc.py

@@ -0,0 +1,164 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import (
+    LCBCodeGenerationDataset,
+    LCBCodeExecutionDataset,
+    LCBTestOutputPredictionDataset,
+    LCBCodeGenerationEvaluator,
+    LCBCodeExecutionEvaluator,
+    LCBTestOutputEvaluator
+)
+from opencompass.datasets.livecodebench import TestOutputPromptConstants
+
+
+lcb_code_generation_reader_cfg = dict(
+    input_columns=[
+        'question_content',
+        'format_prompt',
+    ],
+    # output_column='evaluation_sample',
+    output_column='question_id',
+)
+
+SYSTEM_MESSAGE_GENERIC = f'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.'
+
+prompt_template = '### Question:\n{question_content}\n\n{format_prompt}' + \
+                    '### Answer: (use the provided format with backticks)\n\n'
+
+
+# Code Generation Tasks
+lcb_code_generation_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt=prompt_template
+                )
+            ]
+        )
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=1024)
+)
+
+lcb_code_generation_eval_cfg = dict(
+    evaluator=dict(
+        type=LCBCodeGenerationEvaluator,
+        num_process_evaluate=4,
+        timeout=6,
+    ),
+    pred_role='BOT',
+)
+
+LCBCodeGeneration_dataset = dict(
+    type=LCBCodeGenerationDataset,
+    abbr='lcb_code_generation',
+    path='opencompass/code_generation_lite',
+    reader_cfg=lcb_code_generation_reader_cfg,
+    infer_cfg=lcb_code_generation_infer_cfg,
+    eval_cfg=lcb_code_generation_eval_cfg
+)
+
+# Code Execution Dataset
+lcb_code_execution_reader_cfg = dict(
+    input_columns=[
+        'prompt',
+    ],
+    output_column='evaluation_sample',
+)
+
+lcb_code_execution_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            begin=[
+                dict(
+                    role='SYSTEM',
+                    fallback_role='HUMAN',
+                    prompt='You are an expert at Python programming, code execution, test case generation, and fuzzing.'
+                ),
+            ],
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='{prompt}'
+                )
+            ]
+        )
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=1024)
+)
+
+lcb_code_execution_eval_cfg = dict(
+    evaluator=dict(
+        type=LCBCodeExecutionEvaluator,
+    ),
+    pred_role='BOT',
+)
+
+LCBCodeExecution_dataset = dict(
+    type=LCBCodeExecutionDataset,
+    abbr='lcb_code_execution',
+    path='opencompass/execution-v2',
+    reader_cfg=lcb_code_execution_reader_cfg,
+    infer_cfg=lcb_code_execution_infer_cfg,
+    eval_cfg=lcb_code_execution_eval_cfg,
+)
+
+# TestOuputput Dataset
+lcb_test_output_reader_cfg = dict(
+    input_columns=[
+        'prompt',
+    ],
+    output_column='evaluation_sample',
+)
+
+system_prompt = 'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.'
+
+lcb_test_output_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            # begin=[
+            #     dict(
+            #         role='SYSTEM',
+            #         prompt=system_prompt
+            #     ),
+            # ],
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='{prompt}'
+                )
+            ]
+        )
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=1024)
+)
+
+lcb_test_output_eval_cfg = dict(
+    evaluator=dict(
+        type=LCBTestOutputEvaluator,
+    ),
+    pred_role='BOT',
+)
+
+LCBTestOutput_dataset = dict(
+    type=LCBTestOutputPredictionDataset,
+    abbr='lcb_test_output',
+    path='opencompass/test_generation',
+    reader_cfg=lcb_test_output_reader_cfg,
+    infer_cfg=lcb_test_output_infer_cfg,
+    eval_cfg=lcb_test_output_eval_cfg,
+)
+
+LCB_datasets = [
+    LCBCodeGeneration_dataset,
+    LCBCodeExecution_dataset,
+    LCBTestOutput_dataset,
+]

opencompass/configs/datasets/livecodebench/livecodebench_gen_b2b0fd.py

@@ -0,0 +1,163 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import (
+    LCBCodeGenerationDataset,
+    LCBCodeExecutionDataset,
+    LCBTestOutputPredictionDataset,
+    LCBCodeGenerationEvaluator,
+    LCBCodeExecutionEvaluator,
+    LCBTestOutputEvaluator
+)
+from opencompass.datasets.livecodebench import TestOutputPromptConstants
+
+
+lcb_code_generation_reader_cfg = dict(
+    input_columns=[
+        'question_content',
+        'format_prompt',
+    ],
+    # output_column='evaluation_sample',
+    output_column='question_id',
+)
+
+SYSTEM_MESSAGE_GENERIC = f'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.'
+
+prompt_template = '### Question:\n{question_content}\n\n{format_prompt}' + \
+                    '### Answer: (use the provided format with backticks)\n\n'
+
+
+# Code Generation Tasks
+lcb_code_generation_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt=prompt_template
+                )
+            ]
+        )
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=1024)
+)
+
+lcb_code_generation_eval_cfg = dict(
+    evaluator=dict(
+        type=LCBCodeGenerationEvaluator,
+        num_process_evaluate=4,
+        timeout=6,
+    ),
+    pred_role='BOT',
+)
+
+LCBCodeGeneration_dataset = dict(
+    type=LCBCodeGenerationDataset,
+    abbr='lcb_code_generation',
+    path='opencompass/code_generation_lite',
+    reader_cfg=lcb_code_generation_reader_cfg,
+    infer_cfg=lcb_code_generation_infer_cfg,
+    eval_cfg=lcb_code_generation_eval_cfg
+)
+
+# Code Execution Dataset
+lcb_code_execution_reader_cfg = dict(
+    input_columns=[
+        'prompt',
+    ],
+    output_column='evaluation_sample',
+)
+
+lcb_code_execution_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            begin=[
+                dict(
+                    role='SYSTEM',
+                    prompt='You are an expert at Python programming, code execution, test case generation, and fuzzing.'
+                ),
+            ],
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='{prompt}'
+                )
+            ]
+        )
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=1024)
+)
+
+lcb_code_execution_eval_cfg = dict(
+    evaluator=dict(
+        type=LCBCodeExecutionEvaluator,
+    ),
+    pred_role='BOT',
+)
+
+LCBCodeExecution_dataset = dict(
+    type=LCBCodeExecutionDataset,
+    abbr='lcb_code_execution',
+    path='opencompass/execution-v2',
+    reader_cfg=lcb_code_execution_reader_cfg,
+    infer_cfg=lcb_code_execution_infer_cfg,
+    eval_cfg=lcb_code_execution_eval_cfg,
+)
+
+# TestOuputput Dataset
+lcb_test_output_reader_cfg = dict(
+    input_columns=[
+        'prompt',
+    ],
+    output_column='evaluation_sample',
+)
+
+system_prompt = 'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.'
+
+lcb_test_output_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            # begin=[
+            #     dict(
+            #         role='SYSTEM',
+            #         prompt=system_prompt
+            #     ),
+            # ],
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt='{prompt}'
+                )
+            ]
+        )
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=1024)
+)
+
+lcb_test_output_eval_cfg = dict(
+    evaluator=dict(
+        type=LCBTestOutputEvaluator,
+    ),
+    pred_role='BOT',
+)
+
+LCBTestOutput_dataset = dict(
+    type=LCBTestOutputPredictionDataset,
+    abbr='lcb_test_output',
+    path='opencompass/test_generation',
+    reader_cfg=lcb_test_output_reader_cfg,
+    infer_cfg=lcb_test_output_infer_cfg,
+    eval_cfg=lcb_test_output_eval_cfg,
+)
+
+LCB_datasets = [
+    LCBCodeGeneration_dataset,
+    LCBCodeExecution_dataset,
+    LCBTestOutput_dataset,
+]

opencompass/configs/datasets/mbpp_plus/deprecated_mbpp_plus_gen_94815c.py

@@ -0,0 +1,64 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import MBPPEvaluator, MBPPPlusDataset
+
+mbpp_plus_reader_cfg = dict(
+    input_columns=['text', 'test_list'], output_column='task_id')
+
+mbpp_plus_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    'You are an expert Python programmer, and here is your task: Write a function to find the shared elements from the given two lists. Your code should pass these tests:\n\n assert similar_elements((3, 4, 5, 6),(5, 7, 4, 10)) == (4, 5)\n assert similar_elements((1, 2, 3, 4),(5, 4, 3, 7)) == (3, 4) \n assert similar_elements((11, 12, 14, 13),(17, 15, 14, 13)) == (13, 14) \n'
+                ),
+                dict(
+                    role='BOT',
+                    prompt=
+                    "[BEGIN]\n 'def similar_elements(test_tup1, test_tup2):\n  return tuple(set(test_tup1) & set(test_tup2))' \n[DONE] \n\n "
+                ),
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    'You are an expert Python programmer, and here is your task: Write a python function to identify non-prime numbers. Your code should pass these tests:\n\n assert is_not_prime(2) == False \n assert is_not_prime(10) == True \n assert is_not_prime(35) == True \n'
+                ),
+                dict(
+                    role='BOT',
+                    prompt=
+                    "[BEGIN]\n 'import math\ndef is_not_prime(n):\n    if n == 1:\n        return True\n    for i in range(2, int(math.sqrt(n))+1):\n        if n % i == 0:\n            return True\n    return False' \n[DONE] \n\n "
+                ),
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    'You are an expert Python programmer, and here is your task: Write a function to find the n largest integers from a given list of numbers, returned in descending order. Your code should pass these tests:\n\n assert heap_queue_largest( [25, 35, 22, 85, 14, 65, 75, 22, 58],3)==[85, 75, 65] \n assert heap_queue_largest( [25, 35, 22, 85, 14, 65, 75, 22, 58],2)==[85, 75] \n assert heap_queue_largest( [25, 35, 22, 85, 14, 65, 75, 22, 58],5)==[85, 75, 65, 58, 35] \n'
+                ),
+                dict(
+                    role='BOT',
+                    prompt=
+                    "[BEGIN]\n 'import heapq as hq\ndef heap_queue_largest(nums: list,n: int) -> list:\n  largest_nums = hq.nlargest(n, nums)\n  return largest_nums' \n[DONE] \n\n "
+                ),
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    'You are an expert Python programmer, and here is your task: {text} Your code should pass these tests:\n\n {test_list}  \n'
+                ),
+                dict(role='BOT', prompt='[BEGIN]\n'),
+            ], )),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512))
+
+mbpp_plus_eval_cfg = dict(evaluator=dict(type=MBPPEvaluator, metric='MBPPPlus'), pred_role='BOT')
+
+mbpp_plus_datasets = [
+    dict(
+        type=MBPPPlusDataset,
+        abbr='mbpp_plus',
+        path='./data/mbpp_plus/mbpp_plus.jsonl',
+        reader_cfg=mbpp_plus_reader_cfg,
+        infer_cfg=mbpp_plus_infer_cfg,
+        eval_cfg=mbpp_plus_eval_cfg)
+]

opencompass/configs/datasets/mbpp_plus/mbpp_plus_gen_0b836a.py

@@ -0,0 +1,64 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import MBPPEvaluator, MBPPPlusDataset
+
+mbpp_plus_reader_cfg = dict(
+    input_columns=['text', 'test_list'], output_column='task_id')
+
+mbpp_plus_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    'You are an expert Python programmer, and here is your task: Write a function to find the shared elements from the given two lists. Your code should pass these tests:\n\n assert similar_elements((3, 4, 5, 6),(5, 7, 4, 10)) == (4, 5)\nassert similar_elements((1, 2, 3, 4),(5, 4, 3, 7)) == (3, 4) \nassert similar_elements((11, 12, 14, 13),(17, 15, 14, 13)) == (13, 14) \n'
+                ),
+                dict(
+                    role='BOT',
+                    prompt=
+                    "[BEGIN]\n 'def similar_elements(test_tup1, test_tup2):\n  return tuple(set(test_tup1) & set(test_tup2))' \n[DONE] \n\n "
+                ),
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    'You are an expert Python programmer, and here is your task: Write a python function to identify non-prime numbers. Your code should pass these tests:\n\n assert is_not_prime(2) == False \nassert is_not_prime(10) == True \nassert is_not_prime(35) == True \n'
+                ),
+                dict(
+                    role='BOT',
+                    prompt=
+                    "[BEGIN]\n 'import math\ndef is_not_prime(n):\n    if n == 1:\n        return True\n    for i in range(2, int(math.sqrt(n))+1):\n        if n % i == 0:\n            return True\n    return False' \n[DONE] \n\n "
+                ),
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    'You are an expert Python programmer, and here is your task: Write a function to find the n largest integers from a given list of numbers, returned in descending order. Your code should pass these tests:\n\n assert heap_queue_largest( [25, 35, 22, 85, 14, 65, 75, 22, 58],3)==[85, 75, 65] \nassert heap_queue_largest( [25, 35, 22, 85, 14, 65, 75, 22, 58],2)==[85, 75] \nassert heap_queue_largest( [25, 35, 22, 85, 14, 65, 75, 22, 58],5)==[85, 75, 65, 58, 35] \n'
+                ),
+                dict(
+                    role='BOT',
+                    prompt=
+                    "[BEGIN]\n 'import heapq as hq\ndef heap_queue_largest(nums: list,n: int) -> list:\n  largest_nums = hq.nlargest(n, nums)\n  return largest_nums' \n[DONE] \n\n "
+                ),
+                dict(
+                    role='HUMAN',
+                    prompt=
+                    'You are an expert Python programmer, and here is your task: {text} Your code should pass these tests:\n\n {test_list}  \n'
+                ),
+                dict(role='BOT', prompt='[BEGIN]\n'),
+            ], )),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512))
+
+mbpp_plus_eval_cfg = dict(evaluator=dict(type=MBPPEvaluator, metric='MBPPPlus'), pred_role='BOT')
+
+mbpp_plus_datasets = [
+    dict(
+        type=MBPPPlusDataset,
+        abbr='mbpp_plus',
+        path='./data/mbpp_plus/mbpp_plus.jsonl',
+        reader_cfg=mbpp_plus_reader_cfg,
+        infer_cfg=mbpp_plus_infer_cfg,
+        eval_cfg=mbpp_plus_eval_cfg)
+]

opencompass/configs/datasets/nq_cn/nqcn_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .nqcn_gen_141737 import nqcn_datasets  # noqa: F401, F403

opencompass/configs/datasets/nq_cn/nqcn_gen_141737.py

@@ -0,0 +1,34 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import NaturalQuestionDatasetCN, NQEvaluatorCN
+
+nqcn_reader_cfg = dict(
+    input_columns=['question'], output_column='answer', train_split='test'
+)
+
+nqcn_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='问题: {question}?\n答案是：'),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+nqcn_eval_cfg = dict(evaluator=dict(type=NQEvaluatorCN), pred_role='BOT')
+
+nqcn_datasets = [
+    dict(
+        abbr='nq_cn',
+        type=NaturalQuestionDatasetCN,
+        path='./data/nq_cn',
+        reader_cfg=nqcn_reader_cfg,
+        infer_cfg=nqcn_infer_cfg,
+        eval_cfg=nqcn_eval_cfg,
+    )
+]

opencompass/configs/datasets/race/race_cot_gen_d95929.py

@@ -0,0 +1,68 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import RaceDataset
+from opencompass.utils.text_postprocessors import (
+    first_option_postprocess,
+)
+
+QUERY_TEMPLATE = """
+Answer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.
+
+Article: {article}
+
+Q: {question}
+
+A. {A}
+B. {B}
+C. {C}
+D. {D}
+""".strip()
+
+race_reader_cfg = dict(
+    input_columns=['article', 'question', 'A', 'B', 'C', 'D'],
+    output_column='answer',
+    train_split='validation',
+    test_split='test',
+)
+
+race_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt=QUERY_TEMPLATE),
+            ]
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+race_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'),
+    pred_role='BOT',
+)
+
+race_datasets = [
+    dict(
+        abbr='race-middle',
+        type=RaceDataset,
+        path='opencompass/race',
+        name='middle',
+        reader_cfg=race_reader_cfg,
+        infer_cfg=race_infer_cfg,
+        eval_cfg=race_eval_cfg,
+    ),
+    dict(
+        abbr='race-high',
+        type=RaceDataset,
+        path='opencompass/race',
+        name='high',
+        reader_cfg=race_reader_cfg,
+        infer_cfg=race_infer_cfg,
+        eval_cfg=race_eval_cfg,
+    ),
+]