File size: 3,623 Bytes
60628c7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 |
from opencompass.openicl.icl_prompt_template import PromptTemplate
from opencompass.openicl.icl_retriever import ZeroRetriever
from opencompass.openicl.icl_inferencer import GenInferencer
from opencompass.datasets.ruler.ruler_niah import RulerNiahDataset
from opencompass.datasets.ruler.ruler_niah import RulerNiahEvaluator
# Ruler Dataset settings
niah_configurations = [
{
'abbr': 'single_1',
'type_haystack': 'repeat',
'type_needle_k': 'words',
'type_needle_v': 'numbers',
'num_needle_k': 1,
'num_needle_v': 1,
'num_needle_q': 1,
},
{
'abbr': 'single_2',
'type_haystack': 'essay',
'type_needle_k': 'words',
'type_needle_v': 'numbers',
'num_needle_k': 1,
'num_needle_v': 1,
'num_needle_q': 1,
},
{
'abbr': 'single_3',
'type_haystack': 'essay',
'type_needle_k': 'words',
'type_needle_v': 'uuids',
'num_needle_k': 1,
'num_needle_v': 1,
'num_needle_q': 1,
},
{
'abbr': 'multikey_1',
'type_haystack': 'essay',
'type_needle_k': 'words',
'type_needle_v': 'numbers',
'num_needle_k': 4,
'num_needle_v': 1,
'num_needle_q': 1,
},
{
'abbr': 'multikey_2',
'type_haystack': 'needle',
'type_needle_k': 'words',
'type_needle_v': 'numbers',
'num_needle_k': 1,
'num_needle_v': 1,
'num_needle_q': 1,
},
{
'abbr': 'multikey_3',
'type_haystack': 'needle',
'type_needle_k': 'uuids',
'type_needle_v': 'uuids',
'num_needle_k': 1,
'num_needle_v': 1,
'num_needle_q': 1,
},
{
'abbr': 'multivalue',
'type_haystack': 'essay',
'type_needle_k': 'words',
'type_needle_v': 'numbers',
'num_needle_k': 1,
'num_needle_v': 4,
'num_needle_q': 1,
},
{
'abbr': 'multiquery',
'type_haystack': 'essay',
'type_needle_k': 'words',
'type_needle_v': 'numbers',
'num_needle_k': 1,
'num_needle_v': 1,
'num_needle_q': 4,
},
]
niah_datasets = []
# NIAH Dataset
base_path = './data/ruler'
file_path = 'PaulGrahamEssays.jsonl'
for index, config in enumerate(niah_configurations):
dataset_dict = {
'abbr': f'ruler_niah_{config["abbr"]}',
'type': RulerNiahDataset,
'base_path': base_path,
'file_path': file_path,
# 'tokenizer_model': model_path,
'tokens_to_generate': 128,
# 'max_seq_length': max_seq_len,
# 'num_samples': NUM_SAMPLES,
'type_haystack': config['type_haystack'],
'type_needle_k': config['type_needle_k'],
'type_needle_v': config['type_needle_v'],
'num_needle_k': config['num_needle_k'],
'num_needle_v': config['num_needle_v'],
'num_needle_q': config['num_needle_q'],
'reader_cfg': dict(input_columns=['prompt'], output_column='answer'),
'infer_cfg': dict(
prompt_template=dict(
type=PromptTemplate,
template=dict(
round=[
dict(role='HUMAN', prompt='{prompt}'),
dict(role='BOT', prompt='{answer}\n'),
]
),
),
retriever=dict(type=ZeroRetriever),
inferencer=dict(type=GenInferencer),
),
'eval_cfg': dict(
evaluator=dict(type=RulerNiahEvaluator),
),
}
niah_datasets.append(dataset_dict)
|