Adjust number of reserved tokens to match the model (#15)
Browse files- Adjust number of reserved tokens to match the model (90f5c284285beb0427de55d72590c2c4b61867ee)
Co-authored-by: Dmytro Dzhulgakov <dzhulgakov@users.noreply.huggingface.co>
- tokenization_kimi.py +1 -1
tokenization_kimi.py
CHANGED
|
@@ -104,7 +104,7 @@ class TikTokenTokenizer(PreTrainedTokenizer):
|
|
| 104 |
self.special_tokens = {
|
| 105 |
special_tokens_mapping.get(i, f"<|reserved_token_{i}|>"): i
|
| 106 |
for i in range(
|
| 107 |
-
num_base_tokens, num_base_tokens + self.num_reserved_special_tokens
|
| 108 |
)
|
| 109 |
}
|
| 110 |
|
|
|
|
| 104 |
self.special_tokens = {
|
| 105 |
special_tokens_mapping.get(i, f"<|reserved_token_{i}|>"): i
|
| 106 |
for i in range(
|
| 107 |
+
num_base_tokens, num_base_tokens + self.num_reserved_special_tokens
|
| 108 |
)
|
| 109 |
}
|
| 110 |
|