fix bos, eos token id

config.json

@@ -4,9 +4,9 @@
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
-  "bos_token_id": 100257,
+  "bos_token_id": 100272,
   "end_token_id": 100257,
-  "eos_token_id": 100257,
+  "eos_token_id": 100273,
   "head_dim": 128,
   "hidden_act": "silu",
   "hidden_size": 1024,
@@ -29,4 +29,4 @@
   "transformers_version": "4.50.3",
   "use_cache": false,
   "vocab_size": 110592
-}
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:decbc37141e9bc7b88f38395070c5261d9b68029973dfd3f3b63380c6165a8f4
+size 1132633202

special_tokens_map.json

@@ -56,14 +56,14 @@
     "<PASSWORD>"
   ],
   "bos_token": {
-    "content": "<|endoftext|>",
+    "content": "<|im_start|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "eos_token": {
-    "content": "<|endofturn|>",
+    "content": "<|im_end|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
@@ -83,4 +83,4 @@
     "rstrip": false,
     "single_word": false
   }
-}
+}

tokenizer_config.json

@@ -490,13 +490,13 @@
     "<KEY>",
     "<PASSWORD>"
   ],
-  "bos_token": "<|endoftext|>",
+  "bos_token": "<|im_start|>",
   "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
   "clean_up_tokenization_spaces": true,
-  "eos_token": "<|endofturn|>",
+  "eos_token": "<|im_end|>",
   "extra_special_tokens": {},
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "<|endoftext|>",
   "tokenizer_class": "GPT2Tokenizer",
   "unk_token": "<|endoftext|>"
-}
+}