Spaces:
Sleeping
Sleeping
| # first, keyword identifiers for selecting prompt templates in scripts: | |
| PF_RAW = "raw" | |
| PF_RAWLINES = "rawlines" | |
| PF_SMUGRI_MT = "smugri_mt" | |
| PF_SMUGRI_LID = "smugri_lid" | |
| PF_ALPACA = "alpaca" | |
| # now the prompt templates themselves, SMUGRI LID / MT template: | |
| SMUGRI_INF_PROMPT_LID = "<|reserved_special_token_12|>{src_segm}<|reserved_special_token_13|>" | |
| _SMUGRI_INF_PROMPT_TMPMID = "<|reserved_special_token_14|>{task} to {tgt_lang}<|reserved_special_token_15|>" | |
| SMUGRI_INF_PROMPT_MT = SMUGRI_INF_PROMPT_LID + "{src_lang}" + _SMUGRI_INF_PROMPT_TMPMID | |
| _SMUGRI_TRAIN_PROMPT_PREF = SMUGRI_INF_PROMPT_LID + "{src_lang}" | |
| _SMUGRI_TRAIN_PROMPT_MID = _SMUGRI_INF_PROMPT_TMPMID + "{tgt_segm}" | |
| _SMUGRI_TRAIN_PROMPT_SUF = "<|reserved_special_token_16|><|end_of_text|>" | |
| SMUGRI_PROMPT_TRAIN_PARA = _SMUGRI_TRAIN_PROMPT_PREF + _SMUGRI_TRAIN_PROMPT_MID + _SMUGRI_TRAIN_PROMPT_SUF | |
| SMUGRI_PROMPT_TRAIN_MONO = _SMUGRI_TRAIN_PROMPT_PREF + _SMUGRI_TRAIN_PROMPT_SUF | |
| # Alpaca instructions prompt template: | |
| ALPACA_PROMPT_INF = ("Below is an instruction that describes a task, paired with an input that provides further context. " | |
| "Write a response that appropriately completes the request.\n\n" | |
| "### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:\n") | |
| ALPACA_PROMPT_TRAIN = (ALPACA_PROMPT_INF + "{output}") | |
| def prep_prompt(data, prompt_format, inference=False): | |
| if prompt_format in {PF_RAW, PF_RAWLINES}: | |
| # data is a string, return it | |
| return data | |
| elif prompt_format in {PF_SMUGRI_MT, PF_SMUGRI_LID}: | |
| # data has src_segm, src_lang, tgt_lang, etc | |
| return _prep_ljmf_entry(data, prompt_format, inference) | |
| elif prompt_format == PF_ALPACA: | |
| # data has instruction and input in it | |
| return _prep_alpaca_entry(data, inference) | |
| else: | |
| raise NotImplementedError(f"Prompt format {prompt_format} is not implemented.") | |
| def _prep_alpaca_entry(entry, inference=False): | |
| fmt = ALPACA_PROMPT_INF if inference else ALPACA_PROMPT_TRAIN | |
| prompt = fmt.format(**entry) | |
| return prompt | |
| def _prep_ljmf_entry(entry, fmt, inference=False): | |
| if inference: | |
| if fmt == PF_SMUGRI_MT: | |
| prompt = SMUGRI_INF_PROMPT_MT.format(**entry) | |
| elif fmt == PF_SMUGRI_LID: | |
| prompt = SMUGRI_INF_PROMPT_LID.format(**entry) | |
| else: | |
| raise NotImplementedError(f"Prompt format {fmt} is not implemented.") | |
| else: | |
| if entry['task'] in {'translate', 'approx-translate'} and entry['tgt_segm'] and entry['tgt_lang']: | |
| prompt = SMUGRI_PROMPT_TRAIN_PARA.format(**entry) | |
| else: | |
| prompt = SMUGRI_PROMPT_TRAIN_MONO.format(**entry) | |
| return prompt | |