Update README.md
Browse files
README.md
CHANGED
|
@@ -95,7 +95,7 @@ translator = ctranslate2.Translator(model_path, device=device)
|
|
| 95 |
tokenizer_for_source_language = sentencepiece.SentencePieceProcessor(sentencepiece_model_path+'/spm.ja.nopretok.model')
|
| 96 |
tokenizer_for_target_language = sentencepiece.SentencePieceProcessor(sentencepiece_model_path+'/spm.en.nopretok.model')
|
| 97 |
|
| 98 |
-
#
|
| 99 |
translated_batch=[tokenizer_for_target_language.decode(tokens.hypotheses[0]).replace('<unk>','') for tokens in translator.translate_batch(source=[tokenizer_for_source_language.encode(text,out_type=str) for text in raw_list],beam_size=5)]
|
| 100 |
assert(len(raw_list)==len(translated_batch))
|
| 101 |
|
|
|
|
| 95 |
tokenizer_for_source_language = sentencepiece.SentencePieceProcessor(sentencepiece_model_path+'/spm.ja.nopretok.model')
|
| 96 |
tokenizer_for_target_language = sentencepiece.SentencePieceProcessor(sentencepiece_model_path+'/spm.en.nopretok.model')
|
| 97 |
|
| 98 |
+
#invoke black magic
|
| 99 |
translated_batch=[tokenizer_for_target_language.decode(tokens.hypotheses[0]).replace('<unk>','') for tokens in translator.translate_batch(source=[tokenizer_for_source_language.encode(text,out_type=str) for text in raw_list],beam_size=5)]
|
| 100 |
assert(len(raw_list)==len(translated_batch))
|
| 101 |
|