Andrija
/

RobertaFastBPE

Model card Files Files and versions

Andrija commited on Jul 12, 2021

Commit

62bec84

·

1 Parent(s): 323cc8f

Update README.md

Files changed (1) hide show

README.md +1 -1

README.md CHANGED Viewed

@@ -4,4 +4,4 @@ tokenizer = RobertaTokenizerFast.from_pretrained('Andrija/RobertaFastBPE', bos_t
 encoded = tokenizer('Stručnjaci te bolnice, predvođeni dr Alisom Lim')
 # {'input_ids': [0, 47541, 34632, 603, 24817, 16, 27540, 6768, 2350, 2803, 3991, 2733, 81, 1], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}
 tokenizer.decode(encoded['input_ids'])
-# &lts&gtStručnjaci te bolnice, predvođeni dr Alisom Lim&lt/s&gt

 encoded = tokenizer('Stručnjaci te bolnice, predvođeni dr Alisom Lim')
 # {'input_ids': [0, 47541, 34632, 603, 24817, 16, 27540, 6768, 2350, 2803, 3991, 2733, 81, 1], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}
 tokenizer.decode(encoded['input_ids'])
+# &lt;s&gt;Stručnjaci te bolnice, predvođeni dr Alisom Lim&lt;/s&gt;