Update README.md
Browse files
README.md
CHANGED
@@ -18,4 +18,6 @@ text = "Hello, this is a test input."
|
|
18 |
ids = tokenizer(text)
|
19 |
print(tokenizer.decode(ids['input_ids'], skip_special_tokens=False))
|
20 |
# <bos>Hello, this is a test input.<EOT>
|
|
|
|
|
21 |
```
|
|
|
18 |
ids = tokenizer(text)
|
19 |
print(tokenizer.decode(ids['input_ids'], skip_special_tokens=False))
|
20 |
# <bos>Hello, this is a test input.<EOT>
|
21 |
+
len(tokenizer)
|
22 |
+
# 65004
|
23 |
```
|