Add files using upload-large-folder tool

opencompass/configs/datasets/CLUE_cmnli/CLUE_cmnli_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .CLUE_cmnli_gen_1abf97 import cmnli_datasets  # noqa: F401, F403

opencompass/configs/datasets/CLUE_cmnli/CLUE_cmnli_gen_1abf97.py

@@ -0,0 +1,43 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CMNLIDatasetV2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+cmnli_reader_cfg = dict(
+    input_columns=['sentence1', 'sentence2'],
+    output_column='label',
+    test_split='train')
+
+cmnli_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                '语句一：“{sentence1}”\n语句二：“{sentence2}”\n请问这两句话是什么关系？\nA. 蕴含\nB. 矛盾\nC. 无关\n请从“A”，“B”，“C”中进行选择。\n答：'
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+cmnli_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+cmnli_datasets = [
+    dict(
+        abbr='cmnli',
+        type=CMNLIDatasetV2,
+        path='opencompass/cmnli-dev',
+        reader_cfg=cmnli_reader_cfg,
+        infer_cfg=cmnli_infer_cfg,
+        eval_cfg=cmnli_eval_cfg,
+    )
+]

opencompass/configs/datasets/CLUE_cmnli/CLUE_cmnli_gen_51e956.py

@@ -0,0 +1,43 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CMNLIDatasetV2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+cmnli_reader_cfg = dict(
+    input_columns=['sentence1', 'sentence2'],
+    output_column='label',
+    test_split='train')
+
+cmnli_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                '阅读文章：{sentence1}\n根据上文，回答如下问题：{sentence2}\nA. 对\nB. 错\nC. 可能\n请从“A”，“B”，“C”中进行选择。\n答：'
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+cmnli_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+cmnli_datasets = [
+    dict(
+        abbr='cmnli',
+        type=CMNLIDatasetV2,
+        path='opencompass/cmnli-dev',
+        reader_cfg=cmnli_reader_cfg,
+        infer_cfg=cmnli_infer_cfg,
+        eval_cfg=cmnli_eval_cfg,
+    )
+]

opencompass/configs/datasets/CLUE_cmnli/CLUE_cmnli_ppl.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .CLUE_cmnli_ppl_fdc6de import cmnli_datasets  # noqa: F401, F403

opencompass/configs/datasets/CLUE_cmnli/CLUE_cmnli_ppl_98dd6e.py

@@ -0,0 +1,34 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CMNLIDataset
+
+cmnli_reader_cfg = dict(
+    input_columns=['sentence1', 'sentence2'],
+    output_column='label',
+    test_split='train')
+
+cmnli_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'contradiction':
+            '阅读文章：{sentence1}\n根据上文，回答如下问题： {sentence2}？\n答：错',
+            'entailment': '阅读文章：{sentence1}\n根据上文，回答如下问题： {sentence2}？\n答：对',
+            'neutral': '如果{sentence1}为真，那么{sentence2}也为真吗?可能'
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+cmnli_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+cmnli_datasets = [
+    dict(
+        abbr='cmnli',
+        type=CMNLIDataset,
+        path='opencompass/cmnli-dev',
+        reader_cfg=cmnli_reader_cfg,
+        infer_cfg=cmnli_infer_cfg,
+        eval_cfg=cmnli_eval_cfg)
+]

opencompass/configs/datasets/CLUE_cmnli/CLUE_cmnli_ppl_ef69e7.py

@@ -0,0 +1,50 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CMNLIDataset
+
+cmnli_reader_cfg = dict(
+    input_columns=['sentence1', 'sentence2'],
+    output_column='label',
+    test_split='train')
+
+cmnli_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'contradiction':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='阅读文章：{sentence1}\n根据上文，回答如下问题：{sentence2}？'),
+                dict(role='BOT', prompt='错')
+            ]),
+            'entailment':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='阅读文章：{sentence1}\n根据上文，回答如下问题：{sentence2}？'),
+                dict(role='BOT', prompt='对')
+            ]),
+            'neutral':
+            dict(round=[
+                dict(
+                    role='HUMAN', prompt='如果{sentence1}为真，那么{sentence2}也为真吗？'),
+                dict(role='BOT', prompt='可能')
+            ]),
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+cmnli_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+cmnli_datasets = [
+    dict(
+        abbr='cmnli',
+        type=CMNLIDataset,
+        path='opencompass/cmnli-dev',
+        reader_cfg=cmnli_reader_cfg,
+        infer_cfg=cmnli_infer_cfg,
+        eval_cfg=cmnli_eval_cfg)
+]

opencompass/configs/datasets/CLUE_cmnli/CLUE_cmnli_ppl_fdc6de.py

@@ -0,0 +1,54 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CMNLIDataset
+
+cmnli_reader_cfg = dict(
+    input_columns=['sentence1', 'sentence2'],
+    output_column='label',
+    test_split='train')
+
+cmnli_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            'contradiction':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='语句一：“{sentence1}”\n语句二：“{sentence2}”\n请问这两句话是什么关系？'
+                ),
+                dict(role='BOT', prompt='矛盾')
+            ]),
+            'entailment':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='语句一：“{sentence1}”\n语句二：“{sentence2}”\n请问这两句话是什么关系？'
+                ),
+                dict(role='BOT', prompt='蕴含')
+            ]),
+            'neutral':
+            dict(round=[
+                dict(
+                    role='HUMAN',
+                    prompt='语句一：“{sentence1}”\n语句二：“{sentence2}”\n请问这两句话是什么关系？'
+                ),
+                dict(role='BOT', prompt='无关')
+            ]),
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+cmnli_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+cmnli_datasets = [
+    dict(
+        abbr='cmnli',
+        type=CMNLIDataset,
+        path='opencompass/cmnli-dev',
+        reader_cfg=cmnli_reader_cfg,
+        infer_cfg=cmnli_infer_cfg,
+        eval_cfg=cmnli_eval_cfg)
+]

opencompass/configs/datasets/FewCLUE_csl/FewCLUE_csl_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .FewCLUE_csl_gen_28b223 import csl_datasets  # noqa: F401, F403

opencompass/configs/datasets/FewCLUE_csl/FewCLUE_csl_gen_28b223.py

@@ -0,0 +1,51 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CslDatasetV2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+csl_reader_cfg = dict(
+    input_columns=['abst', 'keywords'],
+    output_column='label',
+)
+
+csl_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                '摘要是对论文内容不加注释和评论的简短陈述，要求扼要地说明研究工作的目的、研究方法和最终结论等。\n关键词是一篇学术论文的核心词汇，一般由一系列名词组成。关键词在全文中应有较高出现频率，且能起到帮助文献检索的作用。\n摘要：{abst}\n关键词：{keywords}\n请问上述关键词是否匹配摘要且符合要求？\nA. 否\nB. 是\n请从”A“，”B“中进行选择。\n答：'
+            )
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+csl_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+csl_datasets = [
+    dict(
+        abbr='csl_dev',
+        type=CslDatasetV2,
+        path='./data/FewCLUE/csl/dev_few_all.json',
+        reader_cfg=csl_reader_cfg,
+        infer_cfg=csl_infer_cfg,
+        eval_cfg=csl_eval_cfg,
+    ),
+    dict(
+        abbr='csl_test',
+        type=CslDatasetV2,
+        path='./data/FewCLUE/csl/test_public.json',
+        reader_cfg=csl_reader_cfg,
+        infer_cfg=csl_infer_cfg,
+        eval_cfg=csl_eval_cfg,
+    ),
+]

opencompass/configs/datasets/FewCLUE_csl/FewCLUE_csl_gen_87f4a8.py

@@ -0,0 +1,51 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CslDatasetV2
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+csl_reader_cfg = dict(
+    input_columns=['abst', 'keywords'],
+    output_column='label',
+)
+
+csl_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                '摘要：{abst}\n关键词：{keywords}\n上述关键词出现在学术期刊中是否恰当？\nA. 否\nB. 是\n请从”A“，”B“中进行选择。\n答：'
+            )
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+csl_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+csl_datasets = [
+    dict(
+        abbr='csl_dev',
+        type=CslDatasetV2,
+        path='./data/FewCLUE/csl/dev_few_all.json',
+        reader_cfg=csl_reader_cfg,
+        infer_cfg=csl_infer_cfg,
+        eval_cfg=csl_eval_cfg,
+    ),
+    dict(
+        abbr='csl_test',
+        type=CslDatasetV2,
+        path='./data/FewCLUE/csl/test_public.json',
+        reader_cfg=csl_reader_cfg,
+        infer_cfg=csl_infer_cfg,
+        eval_cfg=csl_eval_cfg,
+    ),
+]

opencompass/configs/datasets/FewCLUE_csl/FewCLUE_csl_ppl.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .FewCLUE_csl_ppl_841b62 import csl_datasets  # noqa: F401, F403

opencompass/configs/datasets/FewCLUE_csl/FewCLUE_csl_ppl_769f8d.py

@@ -0,0 +1,45 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CslDataset
+
+csl_reader_cfg = dict(
+    input_columns=['abst', 'keywords'], output_column='label')
+
+csl_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0:
+            dict(round=[dict(role='HUMAN', prompt='摘要：{abst}')]),
+            1:
+            dict(
+                round=[dict(role='HUMAN', prompt='摘要：{abst}\n关键词：{keywords}')
+                       ]),
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+csl_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+csl_datasets = [
+    dict(
+        type=CslDataset,
+        path='json',
+        abbr='csl_dev',
+        data_files='./data/FewCLUE/csl/dev_few_all.json',
+        split='train',
+        reader_cfg=csl_reader_cfg,
+        infer_cfg=csl_infer_cfg,
+        eval_cfg=csl_eval_cfg),
+    dict(
+        type=CslDataset,
+        path='json',
+        abbr='csl_test',
+        data_files='./data/FewCLUE/csl/test_public.json',
+        split='train',
+        reader_cfg=csl_reader_cfg,
+        infer_cfg=csl_infer_cfg,
+        eval_cfg=csl_eval_cfg)
+]

opencompass/configs/datasets/FewCLUE_csl/FewCLUE_csl_ppl_841b62.py

@@ -0,0 +1,41 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CslDataset
+
+csl_reader_cfg = dict(
+    input_columns=['abst', 'keywords'], output_column='label')
+
+csl_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0: '摘要：{abst}',
+            1: '摘要：{abst}\n关键词：{keywords}'
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+csl_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+csl_datasets = [
+    dict(
+        type=CslDataset,
+        path='json',
+        abbr='csl_dev',
+        data_files='./data/FewCLUE/csl/dev_few_all.json',
+        split='train',
+        reader_cfg=csl_reader_cfg,
+        infer_cfg=csl_infer_cfg,
+        eval_cfg=csl_eval_cfg),
+    dict(
+        type=CslDataset,
+        path='json',
+        abbr='csl_test',
+        data_files='./data/FewCLUE/csl/test_public.json',
+        split='train',
+        reader_cfg=csl_reader_cfg,
+        infer_cfg=csl_infer_cfg,
+        eval_cfg=csl_eval_cfg)
+]

opencompass/configs/datasets/Xsum/Xsum_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .Xsum_gen_31397e import Xsum_datasets  # noqa: F401, F403

opencompass/configs/datasets/Xsum/Xsum_gen_31397e.py

@@ -0,0 +1,39 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import RougeEvaluator
+from opencompass.datasets import XsumDataset
+
+Xsum_reader_cfg = dict(input_columns=['dialogue'], output_column='summary')
+
+Xsum_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=
+                'Document：{dialogue}\nBased on the previous text, provide a brief single summary:'
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+Xsum_eval_cfg = dict(
+    evaluator=dict(type=RougeEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type='Xsum'),
+)
+
+Xsum_datasets = [
+    dict(
+        type=XsumDataset,
+        abbr='Xsum',
+        path='opencompass/xsum',
+        reader_cfg=Xsum_reader_cfg,
+        infer_cfg=Xsum_infer_cfg,
+        eval_cfg=Xsum_eval_cfg,
+    )
+]

opencompass/configs/datasets/Xsum/Xsum_gen_8ea5f8.py

@@ -0,0 +1,30 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import RougeEvaluator
+from opencompass.datasets import XsumDataset, Xsum_postprocess
+
+Xsum_reader_cfg = dict(input_columns=['dialogue'], output_column='summary')
+
+Xsum_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template='Document：{dialogue}\n'
+        'Based on the previous text, provide a brief single summary:'),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+Xsum_eval_cfg = dict(
+    evaluator=dict(type=RougeEvaluator),
+    pred_postprocessor=dict(type=Xsum_postprocess),
+)
+
+Xsum_datasets = [
+    dict(
+        type=XsumDataset,
+        abbr='Xsum',
+        path='opencompass/xsum',
+        reader_cfg=Xsum_reader_cfg,
+        infer_cfg=Xsum_infer_cfg,
+        eval_cfg=Xsum_eval_cfg)
+]

opencompass/configs/datasets/calm/README.md

@@ -0,0 +1,117 @@
+# CaLM Lite
+**CaLM Lite** is a lightweight version of CaLM.
+
+**Ca**usal evaluation of **L**anguage **M**odels (CaLM), to the best of our knowledge, is the first comprehensive benchmark for evaluating the causal reasoning capabilities of language models. The CaLM framework establishes a foundational taxonomy consisting of four modules: causal target (i.e., what to evaluate), adaptation (i.e., how to obtain the results), metric (i.e., how to measure the results), and error (i.e., how to analyze the bad results).
+
+<div align="center">
+
+[🌐 Website](https://opencausalab.github.io/CaLM) |
+[📃 Report](https://arxiv.org/abs/2405.00622) |[ 🎆 Github](https://github.com/OpenCausaLab/CaLM) | 📧 Welcome to join us by email at [email protected]
+</div>
+
+## Quick Start
+### Data Preparation
+Download dataset to data/ folder.
+```
+wget https://github.com/OpenCausaLab/CaLM/releases/download/v1.0.0.lite/calm.zip
+unzip calm.zip
+```
+### Run Model and Infer
+To obtain a concise output with only the average information for all tasks, use:
+
+```
+python run.py --models YOUR_MODEL --datasets calm --summarizer calm
+```
+
+If you want detailed information for each task, use:
+
+```
+python run.py --models YOUR_MODEL --datasets calm
+```
+
+The `--summarizer calm` flag in the first command is used to generate a summarized output, while omitting it in the second command will provide task-specific details.
+## Available Causal Tasks
+We provide 92 tasks for causal evaluation, stored in the `data/calm` folder. For more information about our causal tasks, refer to [tasks](https://github.com/OpenCausaLab/CaLM/blob/main/documents/tasks.md).
+The directory structure is:
+
+```
+├── calm
+| ├── association
+| ├── causal_discovery # Rung of the causal ladder
+| │ ├── abstract_reasoning # Causal scenario
+| │ │ ├── AR-B_CaLM-AR_CN.json # Causal task
+| │ | └── AR-B_CaLM-AR_EN.json # Causal task
+| │ └── ...
+| └── ...
+└── ...
+```
+
+## Dataset
+- **Dataset size**: CaLM Lite leverages a light dataset of **9200**, while CaLM uses a significantly larger dataset of 126,334. The table below details the English dataset composition, with the Chinese version structured identically.
+- **Dataset configuration**: We prioritize balance in our dataset for **binary classification** and **choice selection** questions. By ensuring an equal number of each GT label, we minimize the risk of introducing bias into the model's testing. For **probability calculation**, CaLM-Lite takes extra attention to balance the number of problems across different causal reasoning processes. (For more details on how causal reasoning process is defined, please refer to Section 9.1.6 of the [paper](https://arxiv.org/abs/2405.00622).)
+- **Efficient evaluation**: For enhanced evaluation efficiency, OpenCompass offers customizable methods. Refer to the [documentation](https://opencompass.org.cn/doc) for guidance on tailoring these methods to your needs.
+
+| Causal ladder | Causal scenario | Subset | Question type | Mode | CaLM Lite | CaLM |
+|---------------|-----------------|--------|---------------|------|-----------|------|
+| Causal discovery | PCD | E-CARE | Binary classification | Natural | 100 | 2000 |
+| Causal discovery | PCD | E-CARE | Choice selection | Natural | 100 | 1000 |
+| Causal discovery | PCD | COPA | Binary classification | Natural | 100 | 2000 |
+| Causal discovery | PCD | COPA | Choice selection | Natural | 100 | 1000 |
+| Causal discovery | ECI | CTB | Binary classification | Natural | 100 | 596 |
+| Causal discovery | ECI | ESC | Binary classification | Natural | 100 | 1000 |
+| Causal discovery | ECI | MAVEN-ERE | Binary classification | Natural | 100 | 1000 |
+| Causal discovery | AR | CaLM-AR | Binary classification | Symbolic | 100 | 1600 |
+| Causal discovery | CA | FP | Binary classification | Symbolic | 100 | 1600 |
+| Causal discovery | CA | FA | Binary classification | Symbolic | 100 | 1600 |
+| Association | CORR | correlation | Binary classification | Natural | 100 | 1476 |
+| Association | EAE | exp-away | Binary classification | Natural | 100 | 168 |
+| Intervention | CB | collider-bias | Binary classification | Natural | 100 | 163 |
+| Intervention | ATE | ATE-natural | Binary classification | Natural | 100 | 1600 |
+| Intervention | ATE | ATE-basic | Probability calculation | Mathematical | 100 | 1600 |
+| Intervention | ATE | ATE-hard | Probability calculation | Mathematical | 100 | 1600 |
+| Intervention | CDE | CDE-natural | Binary classification | Natural | 100 | 1600 |
+| Intervention | CDE | CDE-basic | Probability calculation | Mathematical | 100 | 1600 |
+| Intervention | CDE | CDE-hard | Probability calculation | Mathematical | 100 | 1600 |
+| Intervention | BAS | backadj | Binary classification | Natural | 100 | 227 |
+| Intervention | BAS | max-BAS | Choice selection | Symbolic | 100 | 1600 |
+| Intervention | BAS | min-BAS | Choice selection | Symbolic | 100 | 1600 |
+| Intervention | BAS | mix-BAS | Choice selection | Symbolic | 100 | 1600 |
+| Intervention | FAS | FAS | Choice selection | Symbolic | 100 | 1600 |
+| Intervention | IV | CaLM-IV | Choice selection | Symbolic | 100 | 1600 |
+| Intervention | CEI | 0.2-UC | Binary classification | Symbolic | 100 | 1600 |
+| Intervention | CEI | 0.4-UC | Binary classification | Symbolic | 100 | 1600 |
+| Intervention | CEI | 0.6-UC | Binary classification | Symbolic | 100 | 1600 |
+| Intervention | CEI | 0.8-UC | Binary classification | Symbolic | 100 | 1600 |
+| Counterfactuals | ETT | ETT-natural | Binary classification | Natural | 100 | 1600 |
+| Counterfactuals | ETT | ETT-basic | Probability calculation | Mathematical | 100 | 1600 |
+| Counterfactuals | ETT | ETT-hard | Probability calculation | Mathematical | 100 | 1600 |
+| Counterfactuals | NDE | NDE-natural | Binary classification | Natural | 100 | 1600 |
+| Counterfactuals | NDE | NDE-basic | Probability calculation | Mathematical | 100 | 1600 |
+| Counterfactuals | NDE | NDE-hard | Probability calculation | Mathematical | 100 | 1600 |
+| Counterfactuals | NIE | NIE-natural | Binary classification | Natural | 100 | 1600 |
+| Counterfactuals | NIE | NIE-basic | Probability calculation | Mathematical | 100 | 1600 |
+| Counterfactuals | NIE | NIE-hard | Probability calculation | Mathematical | 100 | 1600 |
+| Counterfactuals | PN | PN-basic | Probability calculation | Mathematical | 100 | 1600 |
+| Counterfactuals | PN | PN-hard | Probability calculation | Mathematical | 100 | 1600 |
+| Counterfactuals | PS | PS-basic | Probability calculation | Mathematical | 100 | 1600 |
+| Counterfactuals | PS | PS-hard | Probability calculation | Mathematical | 100 | 1600 |
+| Counterfactuals | AC | causal judgement | Binary classification | Natural | 100 | 187 |
+| Counterfactuals | CR | CRASS | Choice selection | Natural | 100 | 274 |
+| Counterfactuals | CR | det-counterfactual | Binary classification | Natural | 100 | 1476 |
+| Counterfactuals | CEG | E-CARE | Open-ended generation | Natural | 100 | 1000 |
+| **Total** | | | | | 4600 | 63167 |
+
+## Available Prompt Styles (Adaptation)
+Basic Prompt is our default setting for efficient evaluation of CaLM Lite, but we provide flexibility for exploring additional prompts through CaLM. If you'd like to explore and compare a wider range of prompts, we encourage you to use CaLM. We provide a comprehensive and easy-to-follow guide to assist you in our [repository](https://github.com/OpenCausaLab/CaLM).
+
+## Citation
+```
+@misc{chen2024causal,
+      title={Causal Evaluation of Language Models},
+      author={Sirui Chen and Bo Peng and Meiqi Chen and Ruiqi Wang and Mengying Xu and Xingyu Zeng and Rui Zhao and Shengjie Zhao and Yu Qiao and Chaochao Lu},
+      year={2024},
+      eprint={2405.00622},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+```

opencompass/configs/datasets/calm/calm.py

@@ -0,0 +1,160 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import CaLMDataset, CaLMEvaluator
+
+task_hiearchy_dict = {
+    # association/
+        # correlation/
+            'CORR-B_correlation_CN':'association/correlation/',
+            'CORR-B_correlation_EN':'association/correlation/',
+        # explaining_away_effect/
+            'EAE-B_exp-away_CN':'association/explaining_away_effect/',
+            'EAE-B_exp-away_EN':'association/explaining_away_effect/',
+    # causal_discovery/
+        # abstract_reasoning/
+            'AR-B_CaLM-AR_CN':'causal_discovery/abstract_reasoning/',
+            'AR-B_CaLM-AR_EN':'causal_discovery/abstract_reasoning/',
+        # causal_attribution/
+            'CA-B_FA_CN':'causal_discovery/causal_attribution/',
+            'CA-B_FA_EN':'causal_discovery/causal_attribution/',
+            'CA-B_FP_CN':'causal_discovery/causal_attribution/',
+            'CA-B_FP_EN':'causal_discovery/causal_attribution/',
+        # event_causality_identification/
+            'ECI-B_CTB_CN':'causal_discovery/event_causality_identification/',
+            'ECI-B_CTB_EN':'causal_discovery/event_causality_identification/',
+            'ECI-B_ESC_CN':'causal_discovery/event_causality_identification/',
+            'ECI-B_ESC_EN':'causal_discovery/event_causality_identification/',
+            'ECI-B_MAVEN-ERE_CN':'causal_discovery/event_causality_identification/',
+            'ECI-B_MAVEN-ERE_EN':'causal_discovery/event_causality_identification/',
+        # pairwise_causal_discovery/
+            'PCD-B_COPA_CN':'causal_discovery/pairwise_causal_discovery/',
+            'PCD-B_COPA_EN':'causal_discovery/pairwise_causal_discovery/',
+            'PCD-B_E-CARE_CN':'causal_discovery/pairwise_causal_discovery/',
+            'PCD-B_E-CARE_EN':'causal_discovery/pairwise_causal_discovery/',
+            'PCD-C_COPA_CN':'causal_discovery/pairwise_causal_discovery/',
+            'PCD-C_COPA_EN':'causal_discovery/pairwise_causal_discovery/',
+            'PCD-C_E-CARE_CN':'causal_discovery/pairwise_causal_discovery/',
+            'PCD-C_E-CARE_EN':'causal_discovery/pairwise_causal_discovery/',
+    # counterfactual/
+        # actual_causality/
+            'AC-B_causal_judgement_CN':'counterfactual/actual_causality/',
+            'AC-B_causal_judgement_EN':'counterfactual/actual_causality/',
+        # causal_explanation_generation/
+            'CEG-O_E-CARE_CN':'counterfactual/causal_explanation_generation/',
+            'CEG-O_E-CARE_EN':'counterfactual/causal_explanation_generation/',
+        # counterfactual_reasoning/
+            'CR-B_det-counterfactual_CN':'counterfactual/counterfactual_reasoning/',
+            'CR-B_det-counterfactual_EN':'counterfactual/counterfactual_reasoning/',
+            'CR-C_CRASS_CN':'counterfactual/counterfactual_reasoning/',
+            'CR-C_CRASS_EN':'counterfactual/counterfactual_reasoning/',
+        # effect_of_the_treatment_on_the_treated/
+            'ETT-B_ETT-natural_CN':'counterfactual/effect_of_the_treatment_on_the_treated/',
+            'ETT-B_ETT-natural_EN':'counterfactual/effect_of_the_treatment_on_the_treated/',
+            'ETT-P_ETT-basic_CN':'counterfactual/effect_of_the_treatment_on_the_treated/',
+            'ETT-P_ETT-basic_EN':'counterfactual/effect_of_the_treatment_on_the_treated/',
+            'ETT-P_ETT-hard_CN':'counterfactual/effect_of_the_treatment_on_the_treated/',
+            'ETT-P_ETT-hard_EN':'counterfactual/effect_of_the_treatment_on_the_treated/',
+        # natural_direct_effect/
+            'NDE-B_NDE-natural_CN':'counterfactual/natural_direct_effect/',
+            'NDE-B_NDE-natural_EN':'counterfactual/natural_direct_effect/',
+            'NDE-P_NDE-basic_CN':'counterfactual/natural_direct_effect/',
+            'NDE-P_NDE-basic_EN':'counterfactual/natural_direct_effect/',
+            'NDE-P_NDE-hard_CN':'counterfactual/natural_direct_effect/',
+            'NDE-P_NDE-hard_EN':'counterfactual/natural_direct_effect/',
+        # natural_indirect_effect/
+            'NIE-B_NIE-natural_CN':'counterfactual/natural_indirect_effect/',
+            'NIE-B_NIE-natural_EN':'counterfactual/natural_indirect_effect/',
+            'NIE-P_NIE-basic_CN':'counterfactual/natural_indirect_effect/',
+            'NIE-P_NIE-basic_EN':'counterfactual/natural_indirect_effect/',
+            'NIE-P_NIE-hard_CN':'counterfactual/natural_indirect_effect/',
+            'NIE-P_NIE-hard_EN':'counterfactual/natural_indirect_effect/',
+        # probability_of_necessity/
+            'PN-P_PN-basic_CN':'counterfactual/probability_of_necessity/',
+            'PN-P_PN-basic_EN':'counterfactual/probability_of_necessity/',
+            'PN-P_PN-hard_CN':'counterfactual/probability_of_necessity/',
+            'PN-P_PN-hard_EN':'counterfactual/probability_of_necessity/',
+        # probability_of_sufficiency/
+            'PS-P_PS-basic_CN':'counterfactual/probability_of_sufficiency/',
+            'PS-P_PS-basic_EN':'counterfactual/probability_of_sufficiency/',
+            'PS-P_PS-hard_CN':'counterfactual/probability_of_sufficiency/',
+            'PS-P_PS-hard_EN':'counterfactual/probability_of_sufficiency/',
+    # intervention/
+        # average_treatment_effect/
+            'ATE-B_ATE-natural_CN':'intervention/average_treatment_effect/',
+            'ATE-B_ATE-natural_EN':'intervention/average_treatment_effect/',
+            'ATE-P_ATE-basic_CN':'intervention/average_treatment_effect/',
+            'ATE-P_ATE-basic_EN':'intervention/average_treatment_effect/',
+            'ATE-P_ATE-hard_CN':'intervention/average_treatment_effect/',
+            'ATE-P_ATE-hard_EN':'intervention/average_treatment_effect/',
+        # backdoor_adjustment_set/
+            'BAS-B_backadj_CN':'intervention/backdoor_adjustment_set/',
+            'BAS-B_backadj_EN':'intervention/backdoor_adjustment_set/',
+            'BAS-C_max-BAS_CN':'intervention/backdoor_adjustment_set/',
+            'BAS-C_max-BAS_EN':'intervention/backdoor_adjustment_set/',
+            'BAS-C_min-BAS_CN':'intervention/backdoor_adjustment_set/',
+            'BAS-C_min-BAS_EN':'intervention/backdoor_adjustment_set/',
+            'BAS-C_mix-BAS_CN':'intervention/backdoor_adjustment_set/',
+            'BAS-C_mix-BAS_EN':'intervention/backdoor_adjustment_set/',
+        # causal_effect_identification/
+            'CEI-B_0.2-UC_CN':'intervention/causal_effect_identification/',
+            'CEI-B_0.2-UC_EN':'intervention/causal_effect_identification/',
+            'CEI-B_0.4-UC_CN':'intervention/causal_effect_identification/',
+            'CEI-B_0.4-UC_EN':'intervention/causal_effect_identification/',
+            'CEI-B_0.6-UC_CN':'intervention/causal_effect_identification/',
+            'CEI-B_0.6-UC_EN':'intervention/causal_effect_identification/',
+            'CEI-B_0.8-UC_CN':'intervention/causal_effect_identification/',
+            'CEI-B_0.8-UC_EN':'intervention/causal_effect_identification/',
+        # collider_bias/
+            'CB-B_collider-bias_CN':'intervention/collider_bias/',
+            'CB-B_collider-bias_EN':'intervention/collider_bias/',
+        # controlled_direct_effect/
+            'CDE-B_CDE-natural_CN':'intervention/controlled_direct_effect/',
+            'CDE-B_CDE-natural_EN':'intervention/controlled_direct_effect/',
+            'CDE-P_CDE-basic_CN':'intervention/controlled_direct_effect/',
+            'CDE-P_CDE-basic_EN':'intervention/controlled_direct_effect/',
+            'CDE-P_CDE-hard_CN':'intervention/controlled_direct_effect/',
+            'CDE-P_CDE-hard_EN':'intervention/controlled_direct_effect/',
+        # frontdoor_adjustment_set/
+            'FAS-C_FAS_CN':'intervention/frontdoor_adjustment_set/',
+            'FAS-C_FAS_EN':'intervention/frontdoor_adjustment_set/',
+        # instrumental_variable/
+            'IV-C_CaLM-IV_CN':'intervention/instrumental_variable/',
+            'IV-C_CaLM-IV_EN':'intervention/instrumental_variable/',}
+
+calm_reader_cfg = dict(
+    input_columns=['question'],
+    output_column='gt_item')
+
+calm_all_sets = list(set(key[:-3] for key in task_hiearchy_dict.keys()))
+
+calm_datasets = []
+for _name in calm_all_sets:
+    for _prompt_style in ['basic','basic-CN']:
+        _task_name = _name + ('_CN' if _prompt_style.endswith('-CN') else '_EN')
+        _path = f'./data/calm/{task_hiearchy_dict[_task_name]}{_task_name}.json'
+
+        calm_infer_cfg = dict(
+            prompt_template=dict(
+                type=PromptTemplate,
+                template='{question}'),
+            retriever=dict(type=ZeroRetriever),
+            inferencer=dict(type=GenInferencer, max_out_len=500))
+
+        calm_eval_cfg = dict(evaluator=dict(
+                type=CaLMEvaluator,
+                core_metrics=True,
+                error_analysis=True,
+                prompt_style=_prompt_style,
+                task=_task_name))
+        calm_datasets.append(
+            dict(
+                abbr=f'calm_{_task_name}',
+                type=CaLMDataset,
+                path=_path,
+                prompt_style=_prompt_style,
+                reader_cfg=calm_reader_cfg,
+                infer_cfg=calm_infer_cfg,
+                eval_cfg=calm_eval_cfg)
+        )
+del _prompt_style, _task_name, _path, _name

opencompass/configs/datasets/cvalues/cvalues_responsibility_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .cvalues_responsibility_gen_543378 import cvalues_datasets  # noqa: F401, F403

opencompass/configs/datasets/cvalues/cvalues_responsibility_gen_543378.py

@@ -0,0 +1,37 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import CValuesDataset
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+cvalues_reader_cfg = dict(
+    input_columns=['prompt'],
+    output_column='label',
+    train_split='train',
+    test_split='train',
+)
+
+cvalues_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[dict(role='HUMAN', prompt='{prompt}请直接给出答案：\n')])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer))
+
+cvalues_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+cvalues_datasets = [
+    dict(
+        abbr='CValues-Responsibility',
+        type=CValuesDataset,
+        path='data/cvalues_responsibility_mc.jsonl',
+        reader_cfg=cvalues_reader_cfg,
+        infer_cfg=cvalues_infer_cfg,
+        eval_cfg=cvalues_eval_cfg)
+]

opencompass/configs/datasets/dingo/dingo_gen.py

@@ -0,0 +1,34 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import DingoDataset, DingoEvaluator
+
+
+dingo_paths = [
+    './data/dingo/en_192.csv',
+    './data/dingo/zh_170.csv',
+]
+
+dingo_datasets = []
+for path in dingo_paths:
+    dingo_reader_cfg = dict(input_columns='input', output_column=None)
+    dingo_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(round=[dict(role='HUMAN', prompt='{input}')])),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer),
+    )
+    dingo_eval_cfg = dict(evaluator=dict(type=DingoEvaluator), pred_role='BOT')
+
+    dingo_datasets.append(
+        dict(
+            abbr='dingo_' + path.split('/')[-1].split('.csv')[0],
+            type=DingoDataset,
+            path=path,
+            reader_cfg=dingo_reader_cfg,
+            infer_cfg=dingo_infer_cfg,
+            eval_cfg=dingo_eval_cfg,
+        ))
+
+datasets = dingo_datasets

opencompass/configs/datasets/hellaswag/README.md

@@ -0,0 +1,69 @@
+# HellaSwag
+
+```bash
+python3 run.py --models hf_internlm2_7b --datasets hellaswag_10shot_ppl_59c85e --debug
+python3 run.py --models hf_internlm2_chat_7b --datasets hellaswag_10shot_gen_e42710 --debug
+```
+
+## Base Models
+
+|          model           |   hellaswag |
+|:------------------------:|------------:|
+|    llama-7b-turbomind    |       26.99 |
+|   llama-13b-turbomind    |       34.21 |
+|   llama-30b-turbomind    |       35.65 |
+|   llama-65b-turbomind    |       44.63 |
+|   llama-2-7b-turbomind   |       29.29 |
+|  llama-2-13b-turbomind   |       45.06 |
+|  llama-2-70b-turbomind   |       55.91 |
+|   llama-3-8b-turbomind   |       50.86 |
+|  llama-3-70b-turbomind   |       80.60 |
+| internlm2-1.8b-turbomind |       44.86 |
+|  internlm2-7b-turbomind  |       89.52 |
+| internlm2-20b-turbomind  |       91.41 |
+|   qwen-1.8b-turbomind    |       38.04 |
+|    qwen-7b-turbomind     |       64.62 |
+|    qwen-14b-turbomind    |       85.88 |
+|    qwen-72b-turbomind    |       90.40 |
+|     qwen1.5-0.5b-hf      |       29.19 |
+|     qwen1.5-1.8b-hf      |       42.32 |
+|      qwen1.5-4b-hf       |       55.89 |
+|      qwen1.5-7b-hf       |       68.51 |
+|      qwen1.5-14b-hf      |       83.86 |
+|      qwen1.5-32b-hf      |       87.28 |
+|      qwen1.5-72b-hf      |       90.41 |
+|   qwen1.5-moe-a2-7b-hf   |       72.42 |
+|    mistral-7b-v0.1-hf    |       42.04 |
+|    mistral-7b-v0.2-hf    |       46.24 |
+|   mixtral-8x7b-v0.1-hf   |       66.22 |
+|  mixtral-8x22b-v0.1-hf   |       79.66 |
+|         yi-6b-hf         |       66.83 |
+|        yi-34b-hf         |       83.83 |
+|   deepseek-7b-base-hf    |       30.42 |
+|   deepseek-67b-base-hf   |       70.75 |
+
+## Chat Models
+
+|             model             |   hellaswag |
+|:-----------------------------:|------------:|
+|     qwen1.5-0.5b-chat-hf      |       29.60 |
+|     qwen1.5-1.8b-chat-hf      |       41.71 |
+|      qwen1.5-4b-chat-hf       |       60.45 |
+|      qwen1.5-7b-chat-hf       |       71.58 |
+|      qwen1.5-14b-chat-hf      |       79.70 |
+|      qwen1.5-32b-chat-hf      |       88.56 |
+|      qwen1.5-72b-chat-hf      |       89.37 |
+|     qwen1.5-110b-chat-hf      |       91.11 |
+|    internlm2-chat-1.8b-hf     |       60.47 |
+|  internlm2-chat-1.8b-sft-hf   |       61.58 |
+|     internlm2-chat-7b-hf      |       84.80 |
+|   internlm2-chat-7b-sft-hf    |       85.21 |
+|     internlm2-chat-20b-hf     |       88.48 |
+|   internlm2-chat-20b-sft-hf   |       88.95 |
+|    llama-3-8b-instruct-hf     |       74.39 |
+|    llama-3-70b-instruct-hf    |       89.07 |
+| llama-3-8b-instruct-lmdeploy  |       73.31 |
+| llama-3-70b-instruct-lmdeploy |       87.28 |
+|  mistral-7b-instruct-v0.1-hf  |       53.00 |
+|  mistral-7b-instruct-v0.2-hf  |       65.72 |
+| mixtral-8x7b-instruct-v0.1-hf |       76.16 |

opencompass/configs/datasets/hellaswag/hellaswag_10shot_gen_e42710.py

@@ -0,0 +1,58 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import FixKRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccwithDetailsEvaluator
+from opencompass.datasets import HellaswagDatasetwithICE
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+hellaswag_reader_cfg = dict(
+    input_columns=['ctx', 'A', 'B', 'C', 'D'],
+    output_column='label',
+    train_split='train',
+    test_split='val',
+)
+
+hellaswag_infer_cfg = dict(
+    ice_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt=f'{{ctx}}\nA) {{A}}\nB) {{B}}\nC) {{C}}\nD) {{D}}\nWhat is the right option?'),
+                dict(role='BOT', prompt='{label}\n'),
+            ]
+        ),
+    ),
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            begin=[
+                dict(role='HUMAN', prompt='Continue the following text without adding any additional information or formatting:\n'),
+                '</E>',
+            ],
+            round=[
+                dict(role='HUMAN', prompt=f'{{ctx}}\nA) {{A}}\nB) {{B}}\nC) {{C}}\nD) {{D}}\nWhat is the right option?'),
+                dict(role='BOT', prompt='{label}\n'),
+            ],
+        ),
+        ice_token='</E>',
+    ),
+    retriever=dict(type=FixKRetriever, fix_id_list=list(range(10))),
+    inferencer=dict(type=GenInferencer),
+)
+
+hellaswag_eval_cfg = dict(
+    evaluator=dict(type=AccwithDetailsEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'),
+)
+
+hellaswag_datasets = [
+    dict(
+        abbr='hellaswag',
+        type=HellaswagDatasetwithICE,
+        path='opencompass/hellaswag_ice',
+        reader_cfg=hellaswag_reader_cfg,
+        infer_cfg=hellaswag_infer_cfg,
+        eval_cfg=hellaswag_eval_cfg,
+    )
+]

opencompass/configs/datasets/hellaswag/hellaswag_10shot_ppl_59c85e.py

@@ -0,0 +1,45 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import FixKRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccwithDetailsEvaluator
+from opencompass.datasets import HellaswagDatasetwithICE
+from opencompass.utils.text_postprocessors import first_capital_postprocess
+
+hellaswag_reader_cfg = dict(
+    input_columns=['ctx', 'A', 'B', 'C', 'D'],
+    output_column='label',
+    train_split='train',
+    test_split='val',
+)
+
+hint = 'Continue the following text without adding any additional information or formatting:'
+question_and_options = '{ctx}\nA) {A}\nB) {B}\nC) {C}\nD) {D}\nWhat is the right option?'
+hellaswag_infer_cfg = dict(
+    ice_template=dict(
+        type=PromptTemplate,
+        template={answer: f'{question_and_options}\n{answer}\n' for answer in ['A', 'B', 'C', 'D']},
+    ),
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={answer: f'{hint}\n</E>{question_and_options}\n{answer}' for answer in ['A', 'B', 'C', 'D']},
+        ice_token='</E>',
+    ),
+    retriever=dict(type=FixKRetriever, fix_id_list=list(range(10))),
+    inferencer=dict(type=PPLInferencer),
+)
+
+hellaswag_eval_cfg = dict(
+    evaluator=dict(type=AccwithDetailsEvaluator),
+    pred_postprocessor=dict(type=first_capital_postprocess),
+)
+
+hellaswag_datasets = [
+    dict(
+        abbr='hellaswag',
+        type=HellaswagDatasetwithICE,
+        path='opencompass/hellaswag_ice',
+        reader_cfg=hellaswag_reader_cfg,
+        infer_cfg=hellaswag_infer_cfg,
+        eval_cfg=hellaswag_eval_cfg,
+    )
+]

opencompass/configs/datasets/hellaswag/hellaswag_clean_ppl.py

@@ -0,0 +1,35 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccContaminationEvaluator
+from opencompass.datasets import HellaswagDatasetClean as HellaswagDataset
+
+hellaswag_reader_cfg = dict(
+    input_columns=['ctx', 'A', 'B', 'C', 'D'],
+    output_column='label')
+
+hellaswag_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            i: dict(round=[
+                dict(role='HUMAN', prompt='{ctx}'),
+                dict(role='BOT', prompt=f"{{{chr(ord('A') + i)}}}"),
+            ])
+            for i in range(4)
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+hellaswag_eval_cfg = dict(evaluator=dict(type=AccContaminationEvaluator),
+                          analyze_contamination=True)
+
+hellaswag_datasets = [
+    dict(
+        abbr='hellaswag',
+        type=HellaswagDataset,
+        path='opencompass/hellaswag',
+        reader_cfg=hellaswag_reader_cfg,
+        infer_cfg=hellaswag_infer_cfg,
+        eval_cfg=hellaswag_eval_cfg)
+]

opencompass/configs/datasets/hellaswag/hellaswag_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .hellaswag_gen_6faab5 import hellaswag_datasets  # noqa: F401, F403

opencompass/configs/datasets/hellaswag/hellaswag_gen_6faab5.py

@@ -0,0 +1,44 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HellaswagDataset_V2
+from opencompass.utils.text_postprocessors import first_option_postprocess
+
+hellaswag_reader_cfg = dict(
+    input_columns=['ctx', 'A', 'B', 'C', 'D'],
+    output_column='label',
+)
+
+hellaswag_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt=('{ctx}\nQuestion: Which ending makes the most sense?\n'
+                        'A. {A}\nB. {B}\nC. {C}\nD. {D}\n'
+                        "You may choose from 'A', 'B', 'C', 'D'.\n"
+                        'Answer:'),
+            ),
+        ]),
+    ),
+    retriever=dict(type=ZeroRetriever, ),
+    inferencer=dict(type=GenInferencer),
+)
+
+hellaswag_eval_cfg = dict(
+    evaluator=dict(type=AccEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=first_option_postprocess, options='ABCD'),
+)
+
+hellaswag_datasets = [
+    dict(
+        abbr='hellaswag',
+        type=HellaswagDataset_V2,
+        path='opencompass/hellaswag',
+        reader_cfg=hellaswag_reader_cfg,
+        infer_cfg=hellaswag_infer_cfg,
+        eval_cfg=hellaswag_eval_cfg)
+]

opencompass/configs/datasets/hellaswag/hellaswag_ppl.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .hellaswag_ppl_47bff9 import hellaswag_datasets  # noqa: F401, F403

opencompass/configs/datasets/hellaswag/hellaswag_ppl_47bff9.py

@@ -0,0 +1,34 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HellaswagDataset
+
+hellaswag_reader_cfg = dict(
+    input_columns=['ctx', 'A', 'B', 'C', 'D'],
+    output_column='label')
+
+hellaswag_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            i: dict(round=[
+                dict(role='HUMAN', prompt='{ctx}'),
+                dict(role='BOT', prompt=f"{{{chr(ord('A') + i)}}}"),
+            ])
+            for i in range(4)
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+hellaswag_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+hellaswag_datasets = [
+    dict(
+        abbr='hellaswag',
+        type=HellaswagDataset,
+        path='opencompass/hellaswag',
+        reader_cfg=hellaswag_reader_cfg,
+        infer_cfg=hellaswag_infer_cfg,
+        eval_cfg=hellaswag_eval_cfg)
+]

opencompass/configs/datasets/hellaswag/hellaswag_ppl_7d7f2d.py

@@ -0,0 +1,33 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HellaswagDataset_V2
+
+hellaswag_reader_cfg = dict(
+    input_columns=['query', 'A', 'B', 'C', 'D'],
+    output_column='label')
+
+hellaswag_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            ans: dict(round=[
+                dict(role='HUMAN', prompt='{ctx}\nQuestion: Which ending makes the most sense?\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer: '),
+                dict(role='BOT', prompt=f'{ans}'),
+            ]) for ans in ['A', 'B', 'C', 'D']
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+hellaswag_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+hellaswag_datasets = [
+    dict(
+        abbr='hellaswag',
+        type=HellaswagDataset_V2,
+        path='opencompass/hellaswag',
+        reader_cfg=hellaswag_reader_cfg,
+        infer_cfg=hellaswag_infer_cfg,
+        eval_cfg=hellaswag_eval_cfg)
+]

opencompass/configs/datasets/hellaswag/hellaswag_ppl_9dbb12.py

@@ -0,0 +1,34 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HellaswagDataset
+
+hellaswag_reader_cfg = dict(
+    input_columns=['ctx', 'A', 'B', 'C', 'D'],
+    output_column='label'
+)
+
+hellaswag_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            0: '{ctx} {A}',
+            1: '{ctx} {B}',
+            2: '{ctx} {C}',
+            3: '{ctx} {D}',
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+hellaswag_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+hellaswag_datasets = [
+    dict(
+        abbr='hellaswag',
+        type=HellaswagDataset,
+        path='opencompass/hellaswag',
+        reader_cfg=hellaswag_reader_cfg,
+        infer_cfg=hellaswag_infer_cfg,
+        eval_cfg=hellaswag_eval_cfg)
+]

opencompass/configs/datasets/hellaswag/hellaswag_ppl_a6e128.py

@@ -0,0 +1,41 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import HellaswagDataset_V3
+
+hellaswag_reader_cfg = dict(
+    input_columns=['query', 'A', 'B', 'C', 'D'],
+    output_column='gold')
+
+hellaswag_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template={
+            '0': dict(
+                round=[dict(role='HUMAN', prompt='{query} {A}')]
+            ),
+            '1': dict(
+                round=[dict(role='HUMAN', prompt='{query} {B}')]
+            ),
+            '2': dict(
+                round=[dict(role='HUMAN', prompt='{query} {C}')]
+            ),
+            '3': dict(
+                round=[dict(role='HUMAN', prompt='{query} {D}')]
+            ),
+        }),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=PPLInferencer))
+
+hellaswag_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
+
+hellaswag_datasets = [
+    dict(
+        abbr='hellaswag',
+        type=HellaswagDataset_V3,
+        path='opencompass/hellaswag',
+        reader_cfg=hellaswag_reader_cfg,
+        infer_cfg=hellaswag_infer_cfg,
+        eval_cfg=hellaswag_eval_cfg)
+]

opencompass/configs/datasets/inference_ppl/README.md

@@ -0,0 +1,26 @@
+# Inference-PPL Datasets
+
+- **Description**: Compute the loss only on the labeled positions, especially used for reasoning corpus.
+- **Datasets**: cn-reasoning-val.jsonl (example datasets, inference-ppl can be generalized to more corpus).
+
+# PPL Computation
+
+$$ \text{ppl} = - \frac{1}{n} \sum_{i=0}^n \sum_{c=0}^{vocab\_size} y_{i,c} \log p_{i,c} \tag{1} $$
+
+where Eq. (1) is the normal mean ppl computation formula, for inference-ppl, we only compute the average score based on pre-labeled position.
+
+# Quick Start
+
+```shell
+cd opencompass
+python run.py configs/eval_inference_ppl.py
+```
+
+# Some results
+
+| Model      | Result |
+| ----------- | ----------- |
+| Qwen1.5-7b    | 0.59       |
+| Qwen1.5-14b   | 0.54        |
+| Llama2-7b | 0.49 |
+| Llama2-13b | 0.43 |

opencompass/configs/datasets/inference_ppl/inference_ppl.py

@@ -0,0 +1,38 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import InferencePPLOnlyInferencer
+from opencompass.openicl.icl_evaluator import AverageInferencePPLEvaluator
+
+from opencompass.datasets import InferencePPLDataset
+
+# Build InferencePPLDataset
+inference_ppl_datasets = []
+
+llm_cmp_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template='{text}',
+    ),
+    # No in-context example, using ZeroRetriever
+    retriever=dict(type=ZeroRetriever),
+    # compute inference-ppl
+    inferencer=dict(type=InferencePPLOnlyInferencer),
+)
+
+# Average the inference-ppl scores
+llm_cmp_eval_cfg = dict(evaluator=dict(type=AverageInferencePPLEvaluator))
+
+inference_ppl_datasets.append(
+    dict(
+        abbr=f'inference-ppl',
+        type=InferencePPLDataset,
+        path='./data/inference_ppl',
+        name='cn-reasoning-val',
+        samples=None,  # Set small samples for testing
+        reader_cfg=dict(
+            input_columns=['text'],
+            output_column=None,
+        ),
+        infer_cfg=llm_cmp_infer_cfg,
+        eval_cfg=llm_cmp_eval_cfg,
+    ))

opencompass/configs/datasets/iwslt2017/iwslt2017_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .iwslt2017_gen_d0ebd1 import iwslt2017_datasets  # noqa: F401, F403

opencompass/configs/datasets/iwslt2017/iwslt2017_gen_69ce16.py

@@ -0,0 +1,32 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import BM25Retriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import BleuEvaluator
+from opencompass.datasets import IWSLT2017Dataset
+from opencompass.utils.text_postprocessors import general_cn_postprocess
+
+iwslt2017_reader_cfg = dict(
+    input_columns='en', output_column='de', train_split='validation')
+
+iwslt2017_infer_cfg = dict(
+    ice_template=dict(type='PromptTemplate',
+                    template='</E>{en} = {de}',
+                    ice_token='</E>'),
+    retriever=dict(type=BM25Retriever, ice_num=1),
+    inferencer=dict(type=GenInferencer))
+
+iwslt2017_eval_cfg = dict(
+    evaluator=dict(type=BleuEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=general_cn_postprocess),
+    dataset_postprocessor=dict(type=general_cn_postprocess))
+
+iwslt2017_datasets = [
+    dict(
+        type=IWSLT2017Dataset,
+        path='iwslt2017',
+        name='iwslt2017-en-de',
+        reader_cfg=iwslt2017_reader_cfg,
+        infer_cfg=iwslt2017_infer_cfg,
+        eval_cfg=iwslt2017_eval_cfg)
+]

opencompass/configs/datasets/iwslt2017/iwslt2017_gen_b4a814.py

@@ -0,0 +1,41 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import BM25Retriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import BleuEvaluator
+from opencompass.datasets import IWSLT2017Dataset
+from opencompass.utils.text_postprocessors import general_cn_postprocess
+
+iwslt2017_reader_cfg = dict(
+    input_columns='en', output_column='de', train_split='validation')
+
+iwslt2017_infer_cfg = dict(
+    ice_template=dict(type='PromptTemplate',
+                    template=dict(
+                        begin=[
+                            dict(role='SYSTEM', fallback_role='HUMAN', prompt='Please translate the following English statements to German:'),
+                            '</E>',
+                        ],
+                        round=[
+                            dict(role='HUMAN', prompt='{en}'),
+                            dict(role='BOT', prompt='{de}'),
+                        ]
+                    ),
+                    ice_token='</E>'),
+    retriever=dict(type=BM25Retriever, ice_num=1),
+    inferencer=dict(type=GenInferencer))
+
+iwslt2017_eval_cfg = dict(
+    evaluator=dict(type=BleuEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=general_cn_postprocess),
+    dataset_postprocessor=dict(type=general_cn_postprocess))
+
+iwslt2017_datasets = [
+    dict(
+        type=IWSLT2017Dataset,
+        path='iwslt2017',
+        name='iwslt2017-en-de',
+        reader_cfg=iwslt2017_reader_cfg,
+        infer_cfg=iwslt2017_infer_cfg,
+        eval_cfg=iwslt2017_eval_cfg)
+]

opencompass/configs/datasets/iwslt2017/iwslt2017_gen_d0ebd1.py

@@ -0,0 +1,39 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import BM25Retriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_evaluator import BleuEvaluator
+from opencompass.datasets import IWSLT2017Dataset
+from opencompass.utils.text_postprocessors import general_cn_postprocess
+
+iwslt2017_reader_cfg = dict(
+    input_columns='en', output_column='de', train_split='validation')
+
+iwslt2017_infer_cfg = dict(
+    ice_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            begin='</E>',
+            round=[
+                dict(role='HUMAN', prompt='Please translate the following English statements to German:\n{en}'),
+                dict(role='BOT', prompt='{de}'),
+            ]
+        ),
+        ice_token='</E>'),
+    retriever=dict(type=BM25Retriever, ice_num=1),
+    inferencer=dict(type=GenInferencer))
+
+iwslt2017_eval_cfg = dict(
+    evaluator=dict(type=BleuEvaluator),
+    pred_role='BOT',
+    pred_postprocessor=dict(type=general_cn_postprocess),
+    dataset_postprocessor=dict(type=general_cn_postprocess))
+
+iwslt2017_datasets = [
+    dict(
+        type=IWSLT2017Dataset,
+        path='iwslt2017',
+        name='iwslt2017-en-de',
+        reader_cfg=iwslt2017_reader_cfg,
+        infer_cfg=iwslt2017_infer_cfg,
+        eval_cfg=iwslt2017_eval_cfg)
+]

opencompass/configs/datasets/lawbench/lawbench_one_shot_gen_002588.py

@@ -0,0 +1,62 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LawBenchDataset
+
+names = [
+    ['1-1', 'article_recitation'],
+    ['1-2', 'knowledge_question_answering'],
+    ['2-1', 'document_proofreading'],
+    ['2-2', 'dispute_focus_identification'],
+    ['2-3', 'marital_disputes_identification'],
+    ['2-4', 'issue_topic_identification'],
+    ['2-5', 'reading_comprehension'],
+    ['2-6', 'named_entity_recognition'],
+    ['2-7', 'opinion_summarization'],
+    ['2-8', 'argument_mining'],
+    ['2-9', 'event_detection'],
+    ['2-10', 'trigger_word_extraction'],
+    ['3-1', 'fact_based_article_prediction'],
+    ['3-2', 'scene_based_article_prediction'],
+    ['3-3', 'charge_prediction'],
+    ['3-4', 'prison_term_prediction_wo_article'],
+    ['3-5', 'prison_term_prediction_w_article'],
+    ['3-6', 'case_analysis'],
+    ['3-7', 'criminal_damages_calculation'],
+    ['3-8', 'consultation'],
+]
+
+lawbench_datasets = []
+for index, name in names:
+    lawbench_reader_cfg = dict(
+        input_columns=['instruction', 'question'],
+        output_column='answer')
+
+    lawbench_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=[
+                    dict(role='HUMAN', prompt='{instruction}\n{question}'),
+                ]
+            ),
+        ),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer, max_out_len=1024)
+    )
+
+    lawbench_eval_cfg = dict(
+        evaluator=dict(type='LawBenchEvaluator_' + index.replace('-', '_'))
+    )
+
+    lawbench_datasets.append(
+        dict(
+            abbr='lawbench-' + index + '-' + name + '-1-shot',
+            type=LawBenchDataset,
+            path='./data/lawbench/one_shot',
+            index=index,
+            reader_cfg=lawbench_reader_cfg,
+            infer_cfg=lawbench_infer_cfg,
+            eval_cfg=lawbench_eval_cfg
+        )
+    )

opencompass/configs/datasets/lawbench/lawbench_zero_shot_gen_002588.py

@@ -0,0 +1,62 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import LawBenchDataset
+
+names = [
+    ['1-1', 'article_recitation'],
+    ['1-2', 'knowledge_question_answering'],
+    ['2-1', 'document_proofreading'],
+    ['2-2', 'dispute_focus_identification'],
+    ['2-3', 'marital_disputes_identification'],
+    ['2-4', 'issue_topic_identification'],
+    ['2-5', 'reading_comprehension'],
+    ['2-6', 'named_entity_recognition'],
+    ['2-7', 'opinion_summarization'],
+    ['2-8', 'argument_mining'],
+    ['2-9', 'event_detection'],
+    ['2-10', 'trigger_word_extraction'],
+    ['3-1', 'fact_based_article_prediction'],
+    ['3-2', 'scene_based_article_prediction'],
+    ['3-3', 'charge_prediction'],
+    ['3-4', 'prison_term_prediction_wo_article'],
+    ['3-5', 'prison_term_prediction_w_article'],
+    ['3-6', 'case_analysis'],
+    ['3-7', 'criminal_damages_calculation'],
+    ['3-8', 'consultation'],
+]
+
+lawbench_datasets = []
+for index, name in names:
+    lawbench_reader_cfg = dict(
+        input_columns=['instruction', 'question'],
+        output_column='answer')
+
+    lawbench_infer_cfg = dict(
+        prompt_template=dict(
+            type=PromptTemplate,
+            template=dict(
+                round=[
+                    dict(role='HUMAN', prompt='{instruction}\n{question}'),
+                ]
+            ),
+        ),
+        retriever=dict(type=ZeroRetriever),
+        inferencer=dict(type=GenInferencer, max_out_len=1024)
+    )
+
+    lawbench_eval_cfg = dict(
+        evaluator=dict(type='LawBenchEvaluator_' + index.replace('-', '_'))
+    )
+
+    lawbench_datasets.append(
+        dict(
+            abbr='lawbench-' + index + '-' + name + '-0-shot',
+            type=LawBenchDataset,
+            path='./data/lawbench/zero_shot',
+            index=index,
+            reader_cfg=lawbench_reader_cfg,
+            infer_cfg=lawbench_infer_cfg,
+            eval_cfg=lawbench_eval_cfg
+        )
+    )

opencompass/configs/datasets/math/README.md

@@ -0,0 +1,69 @@
+# MATH
+
+```bash
+python3 run.py --models hf_internlm2_7b --datasets math_4shot_base_gen_db136b --debug
+python3 run.py --models hf_internlm2_chat_7b --datasets math_0shot_gen_393424 --debug
+```
+
+## Base Models
+
+|          model           |   math |
+|:------------------------:|-------:|
+|    llama-7b-turbomind    |   2.94 |
+|   llama-13b-turbomind    |   3.84 |
+|   llama-30b-turbomind    |   6.54 |
+|   llama-65b-turbomind    |  10.66 |
+|   llama-2-7b-turbomind   |   3.58 |
+|  llama-2-13b-turbomind   |   5.30 |
+|  llama-2-70b-turbomind   |  13.26 |
+|   llama-3-8b-turbomind   |  16.42 |
+|  llama-3-70b-turbomind   |  39.64 |
+| internlm2-1.8b-turbomind |   9.42 |
+|  internlm2-7b-turbomind  |  25.16 |
+| internlm2-20b-turbomind  |  32.24 |
+|   qwen-1.8b-turbomind    |   6.30 |
+|    qwen-7b-turbomind     |  15.56 |
+|    qwen-14b-turbomind    |  30.38 |
+|    qwen-72b-turbomind    |  44.18 |
+|     qwen1.5-0.5b-hf      |   4.16 |
+|     qwen1.5-1.8b-hf      |  11.32 |
+|      qwen1.5-4b-hf       |  17.50 |
+|      qwen1.5-7b-hf       |  17.34 |
+|      qwen1.5-14b-hf      |  36.18 |
+|      qwen1.5-32b-hf      |  45.74 |
+|      qwen1.5-72b-hf      |  41.56 |
+|   qwen1.5-moe-a2-7b-hf   |  27.96 |
+|    mistral-7b-v0.1-hf    |  13.44 |
+|    mistral-7b-v0.2-hf    |  12.74 |
+|   mixtral-8x7b-v0.1-hf   |  29.46 |
+|  mixtral-8x22b-v0.1-hf   |  41.82 |
+|         yi-6b-hf         |   6.60 |
+|        yi-34b-hf         |  18.80 |
+|   deepseek-7b-base-hf    |   4.66 |
+|   deepseek-67b-base-hf   |  18.76 |
+
+## Chat Models
+
+|             model             |   math |
+|:-----------------------------:|-------:|
+|     qwen1.5-0.5b-chat-hf      |   0.56 |
+|     qwen1.5-1.8b-chat-hf      |   4.94 |
+|      qwen1.5-4b-chat-hf       |   7.34 |
+|      qwen1.5-7b-chat-hf       |  22.14 |
+|      qwen1.5-14b-chat-hf      |  32.22 |
+|      qwen1.5-32b-chat-hf      |  41.80 |
+|      qwen1.5-72b-chat-hf      |  45.22 |
+|     qwen1.5-110b-chat-hf      |  54.38 |
+|    internlm2-chat-1.8b-hf     |  14.06 |
+|  internlm2-chat-1.8b-sft-hf   |  13.10 |
+|     internlm2-chat-7b-hf      |  28.08 |
+|   internlm2-chat-7b-sft-hf    |  27.60 |
+|     internlm2-chat-20b-hf     |  34.68 |
+|   internlm2-chat-20b-sft-hf   |  32.54 |
+|    llama-3-8b-instruct-hf     |  27.50 |
+|    llama-3-70b-instruct-hf    |  47.52 |
+| llama-3-8b-instruct-lmdeploy  |  27.42 |
+| llama-3-70b-instruct-lmdeploy |  46.90 |
+|  mistral-7b-instruct-v0.1-hf  |   8.48 |
+|  mistral-7b-instruct-v0.2-hf  |  10.82 |
+| mixtral-8x7b-instruct-v0.1-hf |  27.02 |

opencompass/configs/datasets/math/deprecated_math_evaluatorv2_gen_265cce.py

@@ -0,0 +1,38 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import MATHDataset, MATHEvaluator, math_postprocess_v2
+
+math_reader_cfg = dict(input_columns=['problem'], output_column='solution')
+
+math_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(role='HUMAN', prompt='Problem:\nFind the domain of the expression $\\frac{{\sqrt{{x-2}}}}{{\sqrt{{5-x}}}}$.}}\nSolution:'),
+            dict(role='BOT', prompt='The expressions inside each square root must be non-negative. Therefore, $x-2 \ge 0$, so $x\ge2$, and $5 - x \ge 0$, so $x \le 5$. Also, the denominator cannot be equal to zero, so $5-x>0$, which gives $x<5$. Therefore, the domain of the expression is $\\boxed{{[2,5)}}$.\nFinal Answer: The final answer is $[2,5)$. I hope it is correct.\n'),
+            dict(role='HUMAN', prompt='Problem:\nIf $\det \mathbf{{A}} = 2$ and $\det \mathbf{{B}} = 12,$ then find $\det (\mathbf{{A}} \mathbf{{B}}).$\nSolution:'),
+            dict(role='BOT', prompt='We have that $\det (\mathbf{{A}} \mathbf{{B}}) = (\det \mathbf{{A}})(\det \mathbf{{B}}) = (2)(12) = \\boxed{{24}}.$\nFinal Answer: The final answer is $24$. I hope it is correct.\n'),
+            dict(role='HUMAN', prompt='Problem:\nTerrell usually lifts two 20-pound weights 12 times. If he uses two 15-pound weights instead, how many times must Terrell lift them in order to lift the same total weight?\nSolution:'),
+            dict(role='BOT', prompt='If Terrell lifts two 20-pound weights 12 times, he lifts a total of $2\cdot 12\cdot20=480$ pounds of weight. If he lifts two 15-pound weights instead for $n$ times, he will lift a total of $2\cdot15\cdot n=30n$ pounds of weight. Equating this to 480 pounds, we can solve for $n$: \\begin{{align*}} 30n&=480\\\\ \Rightarrow\qquad n&=480/30=\\boxed{{16}} \end{{align*}}\nFinal Answer: The final answer is $16$. I hope it is correct.\n'),
+            dict(role='HUMAN', prompt='Problem:\nIf the system of equations: \\begin{{align*}} 6x-4y&=a,\\\\ 6y-9x &=b. \end{{align*}}has a solution $(x, y)$ where $x$ and $y$ are both nonzero, find $\\frac{{a}}{{b}},$ assuming $b$ is nonzero.\nSolution:'),
+            dict(role='BOT', prompt='If we multiply the first equation by $-\\frac{{3}}{{2}}$, we obtain $$6y-9x=-\\frac{{3}}{{2}}a.$$Since we also know that $6y-9x=b$, we have $$-\\frac{{3}}{{2}}a=b\Rightarrow\\frac{{a}}{{b}}=\\boxed{{-\\frac{{2}}{{3}}}}.$$\nFinal Answer: The final answer is $-\\frac{{2}}{{3}}$. I hope it is correct.\n'),
+            dict(role='HUMAN', prompt='Problem:\n{problem}\nSolution:\n'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512))
+
+# postprocess v2
+math_eval_cfg = dict(
+    evaluator=dict(type=MATHEvaluator, version='v2'),
+    pred_postprocessor=dict(type=math_postprocess_v2))
+
+math_datasets = [
+    dict(
+        type=MATHDataset,
+        abbr='math',
+        path='./data/math/math.json',
+        reader_cfg=math_reader_cfg,
+        infer_cfg=math_infer_cfg,
+        eval_cfg=math_eval_cfg)
+]

opencompass/configs/datasets/math/math_0shot_llm_judge_gen_393424.py

@@ -0,0 +1,78 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import MATHDataset, MATHEvaluator, math_postprocess_v2, GaoKaoMATHEvaluator
+from opencompass.utils.model_postprocessors import naive_model_postprocess, xfinder_postprocess
+from opencompass.utils.postprocessors.naive import MATH_NAVIE_PROMPT_TEMPLATE
+
+# ----------------------------- Eval Parameters -----------------------------
+## Postprocess function
+post_func = 're' # 're', 'xfinder_model', 'naive_model'
+
+## Evalute function
+eval_func = 'naive_model' # 're', 'naive_model'
+
+## Model api url
+xfinder_url = 'http://0.0.0.0:23333/v1' # for 'xFinder-qwen1505' if post_func is 'xfinder_model'
+naive_model_name = 'Qwen/Qwen2.5-72B-Instruct' # replace with your model name
+naive_model_url = ['http://22.8.6.22:23333/v1', 'http://22.8.67.84:23333/v1', 'http://22.8.72.81:23333/v1', 'http://22.9.42.143:23333/v1'] # Multi-apis for accerlation
+
+# ----------------------------- Detailed Config -----------------------------
+
+math_reader_cfg = dict(input_columns=['problem'], output_column='solution')
+
+math_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt='{problem}\nPlease reason step by step, and put your final answer within \\boxed{}.'),
+            ]
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=1024),
+)
+
+if post_func == 're':
+    pred_postprocessor = dict(type=math_postprocess_v2)
+elif post_func == 'xfinder_model':
+    pred_postprocessor = dict(
+        type=xfinder_postprocess,
+        question_type='math',
+        model_name='xFinder-qwen1505',
+        num_processes=128,
+        api_url=xfinder_url,
+    )
+elif post_func == 'naive_model':
+    pred_postprocessor = dict(
+        type=naive_model_postprocess,
+        custom_instruction=MATH_NAVIE_PROMPT_TEMPLATE,
+        model_name=naive_model_name,
+        num_processes=64,
+        api_url=naive_model_url,
+    )
+
+if eval_func == 're':
+    evaluator = dict(type=MATHEvaluator, version='v2')
+elif eval_func == 'naive_model':
+    evaluator = dict(
+        type=GaoKaoMATHEvaluator,
+        model_name=naive_model_name,
+        url=naive_model_url,
+    )
+
+math_eval_cfg = dict(
+    evaluator=evaluator, pred_postprocessor=pred_postprocessor,
+)
+
+math_datasets = [
+    dict(
+        type=MATHDataset,
+        abbr='math',
+        path='opencompass/math',
+        reader_cfg=math_reader_cfg,
+        infer_cfg=math_infer_cfg,
+        eval_cfg=math_eval_cfg,
+    )
+]

opencompass/configs/datasets/math/math_agent_evaluatorv2_gen_0c1b4e.py

@@ -0,0 +1,100 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import AgentInferencer
+from opencompass.datasets import (
+    MATHDataset, MATHAgentEvaluator, math_postprocess_v2
+)
+
+# use pal format but not perform well
+math_reader_cfg = dict(input_columns=['problem'], output_column='solution')
+
+math_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                # # ################################### NEW SHOT ###################################
+                dict(role='HUMAN', prompt='Find the coefficient of $x^3$ when $3(x^2 - x^3+x) +3(x +2x^3- 3x^2 + 3x^5+x^3) -5(1+x-4x^3 - x^2)$ is simplifie.'),
+                dict(role='BOT', prompt="""Tool:PythonInterpreter
+Tool Input:```python
+from sympy import symbols, simplify
+
+def solution():
+    x = symbols('x')
+    expr = 3*(x**2 - x**3 + x) + 3*(x + 2*x**3 - 3*x**2 + 3*x**5 + x**3) - 5*(1 + x - 4*x**3 - x**2)
+    simplified_expr = simplify(expr)
+
+    x3_coefficient = simplified_expr.as_coefficients_dict()[x**3]
+    result = x3_coefficient
+    return result
+```"""),
+                dict(role='SYSTEM', prompt='Response:26'),
+                dict(role='BOT', prompt='FinalAnswer: The final answer is $26$. I hope it is correct.'),
+                dict(role='HUMAN', prompt='The surface area of a sphere with radius $r$ is $4\pi r^2$. Including the area of its circular base, what is the total surface area of a hemisphere with radius 6 cm? Express your answer in terms of $\pi$.'),
+                dict(role='BOT', prompt="""Tool:PythonInterpreter
+Tool Input:```python
+import math
+
+def solution():
+    radius = 6
+
+    # Surface area of the hemisphere
+    hemisphere_area = 2 * math.pi * radius**2
+
+    # Area of the circular base
+    base_area = math.pi * radius**2
+
+    # Total surface area
+    total_surface_area = hemisphere_area + base_area
+
+    # Formatting the result in LaTeX
+    result = r'{}\pi'.format(total_surface_area / math.pi)
+    return result
+```"""),
+                dict(role='SYSTEM', prompt='Response:108.0\\pi'),
+                dict(role='BOT', prompt='FinalAnswer: The final answer is $108.0\pi$. I hope it is correct.'),
+                dict(role='HUMAN', prompt='Monica tosses a fair 6-sided die.  If the roll is a prime number, then she wins that amount of dollars (so that, for example, if she rolls 3, then she wins 3 dollars).  If the roll is composite, she wins nothing. Otherwise, she loses 3 dollars. What is the expected value of her winnings on one die toss? Express your answer as a dollar value to the nearest cent.'),
+                dict(role='BOT', prompt="""Tool:PythonInterpreter
+Tool Input:```python
+def solution():
+    # Probabilities of each outcome
+    prime_prob = 1 / 6
+    composite_prob = 1 / 3
+    otherwise_prob = 1 / 6
+
+    # Expected value of each outcome
+    prime_expected_value = (2 * prime_prob) + (3 * prime_prob) + (5 * prime_prob)
+    composite_expected_value = 0 * composite_prob
+    otherwise_expected_value = -3 * otherwise_prob
+
+    # Total expected value
+    total_expected_value = prime_expected_value + composite_expected_value + otherwise_expected_value
+
+    # Dollar value to the nearest cent
+    result = "{:.2f}".format(total_expected_value)
+    return result
+```"""),
+                dict(role='SYSTEM', prompt='Response:1.17'),
+                dict(role='BOT', prompt='FinalAnswer: The final answer is $1.17$. I hope it is correct.'),
+                dict(role='HUMAN', prompt='{problem}'),
+            ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=AgentInferencer),
+)
+
+math_eval_cfg = dict(
+    evaluator=dict(
+        type=MATHAgentEvaluator,
+        version='v2'),
+    pred_postprocessor=dict(type=math_postprocess_v2))
+
+math_datasets = [
+    dict(
+        abbr='math-agent',
+        type=MATHDataset,
+        path='opencompass/math',
+        reader_cfg=math_reader_cfg,
+        infer_cfg=math_infer_cfg,
+        eval_cfg=math_eval_cfg,
+    )
+]

opencompass/configs/datasets/math/math_agent_gen_0c1b4e.py

@@ -0,0 +1,99 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import AgentInferencer
+from opencompass.datasets import (
+    MATHDataset, MATHAgentEvaluator, math_postprocess
+)
+
+# use pal format but not perform well
+math_reader_cfg = dict(input_columns=['problem'], output_column='solution')
+
+math_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                # # ################################### NEW SHOT ###################################
+                dict(role='HUMAN', prompt='Find the coefficient of $x^3$ when $3(x^2 - x^3+x) +3(x +2x^3- 3x^2 + 3x^5+x^3) -5(1+x-4x^3 - x^2)$ is simplifie.'),
+                dict(role='BOT', prompt="""Tool:PythonInterpreter
+Tool Input:```python
+from sympy import symbols, simplify
+
+def solution():
+    x = symbols('x')
+    expr = 3*(x**2 - x**3 + x) + 3*(x + 2*x**3 - 3*x**2 + 3*x**5 + x**3) - 5*(1 + x - 4*x**3 - x**2)
+    simplified_expr = simplify(expr)
+
+    x3_coefficient = simplified_expr.as_coefficients_dict()[x**3]
+    result = x3_coefficient
+    return result
+```"""),
+                dict(role='SYSTEM', prompt='Response:26'),
+                dict(role='BOT', prompt='FinalAnswer: The final answer is $26$. I hope it is correct.'),
+                dict(role='HUMAN', prompt='The surface area of a sphere with radius $r$ is $4\pi r^2$. Including the area of its circular base, what is the total surface area of a hemisphere with radius 6 cm? Express your answer in terms of $\pi$.'),
+                dict(role='BOT', prompt="""Tool:PythonInterpreter
+Tool Input:```python
+import math
+
+def solution():
+    radius = 6
+
+    # Surface area of the hemisphere
+    hemisphere_area = 2 * math.pi * radius**2
+
+    # Area of the circular base
+    base_area = math.pi * radius**2
+
+    # Total surface area
+    total_surface_area = hemisphere_area + base_area
+
+    # Formatting the result in LaTeX
+    result = r'{}\pi'.format(total_surface_area / math.pi)
+    return result
+```"""),
+                dict(role='SYSTEM', prompt='Response:108.0\\pi'),
+                dict(role='BOT', prompt='FinalAnswer: The final answer is $108.0\pi$. I hope it is correct.'),
+                dict(role='HUMAN', prompt='Monica tosses a fair 6-sided die.  If the roll is a prime number, then she wins that amount of dollars (so that, for example, if she rolls 3, then she wins 3 dollars).  If the roll is composite, she wins nothing. Otherwise, she loses 3 dollars. What is the expected value of her winnings on one die toss? Express your answer as a dollar value to the nearest cent.'),
+                dict(role='BOT', prompt="""Tool:PythonInterpreter
+Tool Input:```python
+def solution():
+    # Probabilities of each outcome
+    prime_prob = 1 / 6
+    composite_prob = 1 / 3
+    otherwise_prob = 1 / 6
+
+    # Expected value of each outcome
+    prime_expected_value = (2 * prime_prob) + (3 * prime_prob) + (5 * prime_prob)
+    composite_expected_value = 0 * composite_prob
+    otherwise_expected_value = -3 * otherwise_prob
+
+    # Total expected value
+    total_expected_value = prime_expected_value + composite_expected_value + otherwise_expected_value
+
+    # Dollar value to the nearest cent
+    result = "{:.2f}".format(total_expected_value)
+    return result
+```"""),
+                dict(role='SYSTEM', prompt='Response:1.17'),
+                dict(role='BOT', prompt='FinalAnswer: The final answer is $1.17$. I hope it is correct.'),
+                dict(role='HUMAN', prompt='{problem}'),
+            ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=AgentInferencer),
+)
+
+math_eval_cfg = dict(
+    evaluator=dict(type=MATHAgentEvaluator),
+    pred_postprocessor=dict(type=math_postprocess),
+)
+
+math_datasets = [
+    dict(
+        abbr='math-agent',
+        type=MATHDataset,
+        path='opencompass/math',
+        reader_cfg=math_reader_cfg,
+        infer_cfg=math_infer_cfg,
+        eval_cfg=math_eval_cfg,
+    )
+]

opencompass/configs/datasets/math/math_evaluatorv2_gen_2f4a71.py

@@ -0,0 +1,56 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import MATHDataset, MATHEvaluator, math_postprocess_v2
+
+math_reader_cfg = dict(input_columns=['problem'], output_column='solution')
+
+math_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template="""\
+Problem:
+Find the domain of the expression $\\frac{{\\sqrt{{x-2}}}}{{\\sqrt{{5-x}}}}$.
+Solution:
+The expressions inside each square root must be non-negative. Therefore, $x-2 \\ge 0$, so $x\\ge2$, and $5 - x \\ge 0$, so $x \\le 5$. Also, the denominator cannot be equal to zero, so $5-x>0$, which gives $x<5$. Therefore, the domain of the expression is $\\boxed{{[2,5)}}$.
+Final Answer: The final answer is $[2,5)$. I hope it is correct.
+
+Problem:
+If $\\det \\mathbf{{A}} = 2$ and $\\det \\mathbf{{B}} = 12,$ then find $\\det (\\mathbf{{A}} \\mathbf{{B}}).$
+Solution:
+We have that $\\det (\\mathbf{{A}} \\mathbf{{B}}) = (\\det \\mathbf{{A}})(\\det \\mathbf{{B}}) = (2)(12) = \\boxed{{24}}.$
+Final Answer: The final answer is $24$. I hope it is correct.
+
+Problem:
+Terrell usually lifts two 20-pound weights 12 times. If he uses two 15-pound weights instead, how many times must Terrell lift them in order to lift the same total weight?
+Solution:
+If Terrell lifts two 20-pound weights 12 times, he lifts a total of $2\\cdot 12\\cdot20=480$ pounds of weight. If he lifts two 15-pound weights instead for $n$ times, he will lift a total of $2\\cdot15\\cdot n=30n$ pounds of weight. Equating this to 480 pounds, we can solve for $n$: \\begin{{align*}} 30n&=480\\\\ \\Rightarrow\\qquad n&=480/30=\\boxed{{16}} \\end{{align*}}
+Final Answer: The final answer is $16$. I hope it is correct.
+
+Problem:
+If the system of equations: \\begin{{align*}} 6x-4y&=a,\\\\ 6y-9x &=b. \\end{{align*}}has a solution $(x, y)$ where $x$ and $y$ are both nonzero, find $\\frac{{a}}{{b}},$ assuming $b$ is nonzero.
+Solution:
+If we multiply the first equation by $-\\frac{{3}}{{2}}$, we obtain $$6y-9x=-\\frac{{3}}{{2}}a.$$Since we also know that $6y-9x=b$, we have $$-\\frac{{3}}{{2}}a=b\\Rightarrow\\frac{{a}}{{b}}=\\boxed{{-\\frac{{2}}{{3}}}}.$$
+Final Answer: The final answer is $-\\frac{{2}}{{3}}$. I hope it is correct.
+
+Problem:
+{problem}
+Solution:"""
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512, stopping_criteria=['Problem']))
+
+# postprocess v2
+math_eval_cfg = dict(
+    evaluator=dict(type=MATHEvaluator, version='v2'),
+    pred_postprocessor=dict(type=math_postprocess_v2))
+
+math_datasets = [
+    dict(
+        type=MATHDataset,
+        abbr='math',
+        path='opencompass/math',
+        reader_cfg=math_reader_cfg,
+        infer_cfg=math_infer_cfg,
+        eval_cfg=math_eval_cfg)
+]

opencompass/configs/datasets/math/math_gen.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+    from .math_gen_265cce import math_datasets  # noqa: F401, F403

opencompass/configs/datasets/math/math_gen_0957ff.py

@@ -0,0 +1,36 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import MATHDataset, MATHEvaluator, math_postprocess
+
+math_reader_cfg = dict(input_columns=['problem'], output_column='solution')
+
+math_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(role='HUMAN', prompt='Problem:\nFind the domain of the expression $\\frac{\sqrt{x-2}}{\sqrt{5-x}}$.}\nSolution:'),
+            dict(role='BOT', prompt='The expressions inside each square root must be non-negative. Therefore, $x-2 \ge 0$, so $x\ge2$, and $5 - x \ge 0$, so $x \le 5$. Also, the denominator cannot be equal to zero, so $5-x>0$, which gives $x<5$. Therefore, the domain of the expression is $\\boxed{[2,5)}$.\nFinal Answer: The final answer is $[2,5)$. I hope it is correct.\n'),
+            dict(role='HUMAN', prompt='Problem:\nIf $\det \mathbf{A} = 2$ and $\det \mathbf{B} = 12,$ then find $\det (\mathbf{A} \mathbf{B}).$\nSolution:'),
+            dict(role='BOT', prompt='We have that $\det (\mathbf{A} \mathbf{B}) = (\det \mathbf{A})(\det \mathbf{B}) = (2)(12) = \\boxed{24}.$\nFinal Answer: The final answer is $24$. I hope it is correct.\n'),
+            dict(role='HUMAN', prompt='Problem:\nTerrell usually lifts two 20-pound weights 12 times. If he uses two 15-pound weights instead, how many times must Terrell lift them in order to lift the same total weight?\nSolution:'),
+            dict(role='BOT', prompt='If Terrell lifts two 20-pound weights 12 times, he lifts a total of $2\cdot 12\cdot20=480$ pounds of weight. If he lifts two 15-pound weights instead for $n$ times, he will lift a total of $2\cdot15\cdot n=30n$ pounds of weight. Equating this to 480 pounds, we can solve for $n$: \\begin{align*} 30n&=480\\\\ \Rightarrow\qquad n&=480/30=\\boxed{16} \end{align*}\nFinal Answer: The final answer is $16$. I hope it is correct.\n'),
+            dict(role='HUMAN', prompt='Problem:\nIf the system of equations: \\begin{align*} 6x-4y&=a,\\\\ 6y-9x &=b. \end{align*}has a solution $(x, y)$ where $x$ and $y$ are both nonzero, find $\\frac{a}{b},$ assuming $b$ is nonzero.\nSolution:'),
+            dict(role='BOT', prompt='If we multiply the first equation by $-\\frac{3}{2}$, we obtain $$6y-9x=-\\frac{3}{2}a.$$Since we also know that $6y-9x=b$, we have $$-\\frac{3}{2}a=b\Rightarrow\\frac{a}{b}=\\boxed{-\\frac{2}{3}}.$$\nFinal Answer: The final answer is $-\\frac{2}{3}$. I hope it is correct.\n'),
+            dict(role='HUMAN', prompt='Problem:\n{problem}\nSolution:\n'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512))
+
+math_eval_cfg = dict(
+    evaluator=dict(type=MATHEvaluator), pred_postprocessor=dict(type=math_postprocess))
+
+math_datasets = [
+    dict(
+        type=MATHDataset,
+        abbr='math',
+        path='opencompass/math',
+        reader_cfg=math_reader_cfg,
+        infer_cfg=math_infer_cfg,
+        eval_cfg=math_eval_cfg)
+]

opencompass/configs/datasets/math/math_gen_265cce.py

@@ -0,0 +1,36 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import MATHDataset, MATHEvaluator, math_postprocess_v2
+
+math_reader_cfg = dict(input_columns=['problem'], output_column='solution')
+
+math_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(role='HUMAN', prompt='Problem:\nFind the domain of the expression $\\frac{{\sqrt{{x-2}}}}{{\sqrt{{5-x}}}}$.}}\nSolution:'),
+            dict(role='BOT', prompt='The expressions inside each square root must be non-negative. Therefore, $x-2 \ge 0$, so $x\ge2$, and $5 - x \ge 0$, so $x \le 5$. Also, the denominator cannot be equal to zero, so $5-x>0$, which gives $x<5$. Therefore, the domain of the expression is $\\boxed{{[2,5)}}$.\nFinal Answer: The final answer is $[2,5)$. I hope it is correct.\n'),
+            dict(role='HUMAN', prompt='Problem:\nIf $\det \mathbf{{A}} = 2$ and $\det \mathbf{{B}} = 12,$ then find $\det (\mathbf{{A}} \mathbf{{B}}).$\nSolution:'),
+            dict(role='BOT', prompt='We have that $\det (\mathbf{{A}} \mathbf{{B}}) = (\det \mathbf{{A}})(\det \mathbf{{B}}) = (2)(12) = \\boxed{{24}}.$\nFinal Answer: The final answer is $24$. I hope it is correct.\n'),
+            dict(role='HUMAN', prompt='Problem:\nTerrell usually lifts two 20-pound weights 12 times. If he uses two 15-pound weights instead, how many times must Terrell lift them in order to lift the same total weight?\nSolution:'),
+            dict(role='BOT', prompt='If Terrell lifts two 20-pound weights 12 times, he lifts a total of $2\cdot 12\cdot20=480$ pounds of weight. If he lifts two 15-pound weights instead for $n$ times, he will lift a total of $2\cdot15\cdot n=30n$ pounds of weight. Equating this to 480 pounds, we can solve for $n$: \\begin{{align*}} 30n&=480\\\\ \Rightarrow\qquad n&=480/30=\\boxed{{16}} \end{{align*}}\nFinal Answer: The final answer is $16$. I hope it is correct.\n'),
+            dict(role='HUMAN', prompt='Problem:\nIf the system of equations: \\begin{{align*}} 6x-4y&=a,\\\\ 6y-9x &=b. \end{{align*}}has a solution $(x, y)$ where $x$ and $y$ are both nonzero, find $\\frac{{a}}{{b}},$ assuming $b$ is nonzero.\nSolution:'),
+            dict(role='BOT', prompt='If we multiply the first equation by $-\\frac{{3}}{{2}}$, we obtain $$6y-9x=-\\frac{{3}}{{2}}a.$$Since we also know that $6y-9x=b$, we have $$-\\frac{{3}}{{2}}a=b\Rightarrow\\frac{{a}}{{b}}=\\boxed{{-\\frac{{2}}{{3}}}}.$$\nFinal Answer: The final answer is $-\\frac{{2}}{{3}}$. I hope it is correct.\n'),
+            dict(role='HUMAN', prompt='Problem:\n{problem}\nSolution:\n'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512))
+
+math_eval_cfg = dict(
+    evaluator=dict(type=MATHEvaluator, version='v2'), pred_postprocessor=dict(type=math_postprocess_v2))
+
+math_datasets = [
+    dict(
+        type=MATHDataset,
+        abbr='math',
+        path='opencompass/math',
+        reader_cfg=math_reader_cfg,
+        infer_cfg=math_infer_cfg,
+        eval_cfg=math_eval_cfg)
+]

opencompass/configs/datasets/math/math_gen_736506.py

@@ -0,0 +1,28 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import MATHInternDataset, MATHInternEvaluator, math_intern_postprocess
+
+math_reader_cfg = dict(input_columns=['problem'], output_column='solution')
+
+math_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(role='HUMAN', prompt="Question: {problem}\nLet's think step by step\nAnswer:")
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512))
+
+math_eval_cfg = dict(
+    evaluator=dict(type=MATHInternEvaluator), pred_postprocessor=dict(type=math_intern_postprocess))
+
+math_datasets = [
+    dict(
+        type=MATHInternDataset,
+        abbr='math',
+        path='opencompass/math',
+        reader_cfg=math_reader_cfg,
+        infer_cfg=math_infer_cfg,
+        eval_cfg=math_eval_cfg)
+]