spin-80k / tokenizer.json
QuatumTech's picture
Fix config hyperparameters for 80k architecture and tie embeddings
759d613 verified
Raw
History Blame Contribute Delete
21.6 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<|unk|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<|pad|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<|bos|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "<|eos|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 4,
"content": "<|im_start|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 5,
"content": "<|im_end|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": true,
"use_regex": true
},
"post_processor": null,
"decoder": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": true,
"use_regex": true
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": "<|unk|>",
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"<|unk|>": 0,
"<|pad|>": 1,
"<|bos|>": 2,
"<|eos|>": 3,
"<|im_start|>": 4,
"<|im_end|>": 5,
"!": 6,
"\"": 7,
"#": 8,
"$": 9,
"%": 10,
"&": 11,
"'": 12,
"(": 13,
")": 14,
"*": 15,
"+": 16,
",": 17,
"-": 18,
".": 19,
"/": 20,
"0": 21,
"1": 22,
"2": 23,
"3": 24,
"4": 25,
"5": 26,
"6": 27,
"7": 28,
"8": 29,
"9": 30,
":": 31,
";": 32,
"<": 33,
"=": 34,
">": 35,
"?": 36,
"@": 37,
"A": 38,
"B": 39,
"C": 40,
"D": 41,
"E": 42,
"F": 43,
"G": 44,
"H": 45,
"I": 46,
"J": 47,
"K": 48,
"L": 49,
"M": 50,
"N": 51,
"O": 52,
"P": 53,
"Q": 54,
"R": 55,
"S": 56,
"T": 57,
"U": 58,
"V": 59,
"W": 60,
"X": 61,
"Y": 62,
"Z": 63,
"[": 64,
"\\": 65,
"]": 66,
"^": 67,
"_": 68,
"`": 69,
"a": 70,
"b": 71,
"c": 72,
"d": 73,
"e": 74,
"f": 75,
"g": 76,
"h": 77,
"i": 78,
"j": 79,
"k": 80,
"l": 81,
"m": 82,
"n": 83,
"o": 84,
"p": 85,
"q": 86,
"r": 87,
"s": 88,
"t": 89,
"u": 90,
"v": 91,
"w": 92,
"x": 93,
"y": 94,
"z": 95,
"{": 96,
"|": 97,
"}": 98,
"~": 99,
"¡": 100,
"¢": 101,
"£": 102,
"¤": 103,
"¥": 104,
"¦": 105,
"§": 106,
"¨": 107,
"©": 108,
"ª": 109,
"«": 110,
"¬": 111,
"®": 112,
"¯": 113,
"°": 114,
"±": 115,
"²": 116,
"³": 117,
"´": 118,
"µ": 119,
"¶": 120,
"·": 121,
"¸": 122,
"¹": 123,
"º": 124,
"»": 125,
"¼": 126,
"½": 127,
"¾": 128,
"¿": 129,
"À": 130,
"Á": 131,
"Â": 132,
"Ã": 133,
"Ä": 134,
"Å": 135,
"Æ": 136,
"Ç": 137,
"È": 138,
"É": 139,
"Ê": 140,
"Ë": 141,
"Ì": 142,
"Í": 143,
"Î": 144,
"Ï": 145,
"Ð": 146,
"Ñ": 147,
"Ò": 148,
"Ó": 149,
"Ô": 150,
"Õ": 151,
"Ö": 152,
"×": 153,
"Ø": 154,
"Ù": 155,
"Ú": 156,
"Û": 157,
"Ü": 158,
"Ý": 159,
"Þ": 160,
"ß": 161,
"à": 162,
"á": 163,
"â": 164,
"ã": 165,
"ä": 166,
"å": 167,
"æ": 168,
"ç": 169,
"è": 170,
"é": 171,
"ê": 172,
"ë": 173,
"ì": 174,
"í": 175,
"î": 176,
"ï": 177,
"ð": 178,
"ñ": 179,
"ò": 180,
"ó": 181,
"ô": 182,
"õ": 183,
"ö": 184,
"÷": 185,
"ø": 186,
"ù": 187,
"ú": 188,
"û": 189,
"ü": 190,
"ý": 191,
"þ": 192,
"ÿ": 193,
"Ā": 194,
"ā": 195,
"Ă": 196,
"ă": 197,
"Ą": 198,
"ą": 199,
"Ć": 200,
"ć": 201,
"Ĉ": 202,
"ĉ": 203,
"Ċ": 204,
"ċ": 205,
"Č": 206,
"č": 207,
"Ď": 208,
"ď": 209,
"Đ": 210,
"đ": 211,
"Ē": 212,
"ē": 213,
"Ĕ": 214,
"ĕ": 215,
"Ė": 216,
"ė": 217,
"Ę": 218,
"ę": 219,
"Ě": 220,
"ě": 221,
"Ĝ": 222,
"ĝ": 223,
"Ğ": 224,
"ğ": 225,
"Ġ": 226,
"ġ": 227,
"Ģ": 228,
"ģ": 229,
"Ĥ": 230,
"ĥ": 231,
"Ħ": 232,
"ħ": 233,
"Ĩ": 234,
"ĩ": 235,
"Ī": 236,
"ī": 237,
"Ĭ": 238,
"ĭ": 239,
"Į": 240,
"į": 241,
"İ": 242,
"ı": 243,
"IJ": 244,
"ij": 245,
"Ĵ": 246,
"ĵ": 247,
"Ķ": 248,
"ķ": 249,
"ĸ": 250,
"Ĺ": 251,
"ĺ": 252,
"Ļ": 253,
"ļ": 254,
"Ľ": 255,
"ľ": 256,
"Ŀ": 257,
"ŀ": 258,
"Ł": 259,
"ł": 260,
"Ń": 261,
"he": 262,
"Ġt": 263,
"Ġa": 264,
"Ġs": 265,
"Ġw": 266,
"nd": 267,
"Ġthe": 268,
"ed": 269,
"Ġand": 270,
"Ġto": 271,
"Ġb": 272,
"in": 273,
"Ġh": 274,
"Ġwa": 275,
"re": 276,
"Ġf": 277,
"ou": 278,
"it": 279,
"Ġl": 280,
"Ġc": 281,
"Ġd": 282,
"Ġhe": 283,
"er": 284,
"Ġm": 285,
"Ġp": 286,
"Ġwas": 287,
"ĠT": 288,
"om": 289,
"Ġo": 290,
"ay": 291,
"is": 292,
"ar": 293,
"il": 294,
"ing": 295,
"Ġg": 296,
"id": 297,
"at": 298,
"en": 299,
"Ġn": 300,
"Ġsa": 301,
"Ġha": 302,
"ĠS": 303,
"ĠThe": 304,
"an": 305,
"im": 306,
"or": 307,
"on": 308,
"Ġit": 309,
"Ġth": 310,
"ll": 311,
"le": 312,
"ĠH": 313,
"Ġher": 314,
"et": 315,
"ot": 316,
"ir": 317,
"ĠShe": 318,
"ĠHe": 319,
"es": 320,
"Ġin": 321,
"ow": 322,
"ut": 323,
"ver": 324,
"ĠThey": 325,
"ck": 326,
"ily": 327,
"Ġu": 328,
"Ġe": 329,
"Ġy": 330,
"ld": 331,
"oo": 332,
"Ġsaid": 333,
"ig": 334,
"Ġ\"": 335,
"Ġr": 336,
"am": 337,
"Ġbe": 338,
"ĠI": 339,
"Ġst": 340,
"Lily": 341,
"ith": 342,
"ke": 343,
"ce": 344,
"pp": 345,
"ve": 346,
"Ġwith": 347,
"Ġshe": 348,
"On": 349,
"Ġon": 350,
"Ġyou": 351,
"ri": 352,
"ked": 353,
"Ġpl": 354,
"Ġhis": 355,
"Ġof": 356,
"nt": 357,
"Ġso": 358,
"ad": 359,
"Ġday": 360,
"very": 361,
"Ġthat": 362,
"ĠLily": 363,
"Ġplay": 364,
"Ġup": 365,
"Ġwe": 366,
"Ġmom": 367,
"Ġhad": 368,
"Ġthey": 369,
"st": 370,
"ĠB": 371,
"el": 372,
"Ġfor": 373,
"un": 374,
"my": 375,
"'s": 376,
"ould": 377,
"Ġli": 378,
"The": 379,
"Ġhapp": 380,
"ent": 381,
"ch": 382,
"itt": 383,
"Ġwant": 384,
"her": 385,
"Ġsh": 386,
"out": 387,
"Ġdo": 388,
"Ġnot": 389,
"end": 390,
"ome": 391,
"all": 392,
"se": 393,
"Ġvery": 394,
"ound": 395,
"ly": 396,
"ime": 397,
"ittle": 398,
"Ġk": 399,
"Ġtime": 400,
"Ġwh": 401,
"Ġlittle": 402,
"Ġthere": 403,
"Ġne": 404,
"Ġis": 405,
"Ġbig": 406,
"ht": 407,
"Ġhappy": 408,
"al": 409,
"ĠM": 410,
"Ġre": 411,
"Ġsm": 412,
"ĠĊ": 413,
"riend": 414,
"Ġfriend": 415,
"ĠIt": 416,
"Ġsaw": 417,
"Ġbut": 418,
".\"": 419,
"ra": 420,
"Ġan": 421,
"Ġas": 422,
"ry": 423,
"One": 424,
"ack": 425,
"ide": 426,
"ake": 427,
"ved": 428,
"'t": 429,
"!\"": 430,
"ug": 431,
"ter": 432,
"Ġlo": 433,
"Ġloo": 434,
"Once": 435,
"Ġwere": 436,
"ĠTim": 437,
"Ġse": 438,
"ore": 439,
"Ġbo": 440,
"Ġtoo": 441,
"Ġhim": 442,
"Ġgo": 443,
"Ġare": 444,
"ec": 445,
"Ġupon": 446,
"ill": 447,
"irl": 448,
"Ġwanted": 449,
"Ġthem": 450,
"ard": 451,
"Ġj": 452,
"Ġgirl": 453,
"Ġout": 454,
"Ġtheir": 455,
"Ġat": 456,
"ind": 457,
"way": 458,
"ur": 459,
"fu": 460,
"Ġdid": 461,
"ĠA": 462,
"Ġsp": 463,
"Ġsmil": 464,
"Ġcould": 465,
"hen": 466,
"Ġhave": 467,
"ain": 468,
"ĠBen": 469,
"Ġwent": 470,
"hed": 471,
"Ġnam": 472,
"Ġcan": 473,
"art": 474,
"rom": 475,
"lp": 476,
"ic": 477,
"Ġex": 478,
"round": 479,
"Ġfriends": 480,
"?\"": 481,
"Ġhelp": 482,
"ood": 483,
"ful": 484,
"Ġkn": 485,
"ĠJ": 486,
"Ġall": 487,
"hing": 488,
"ight": 489,
"um": 490,
"ark": 491,
"Ġfun": 492,
"one": 493,
"Ġsay": 494,
"Ġback": 495,
"You": 496,
"Ġnamed": 497,
"are": 498,
"Ġcl": 499,
"ĠTom": 500,
"Ġno": 501,
"Ġle": 502,
"Ġloved": 503,
"op": 504,
"elt": 505,
"ĠTimmy": 506,
"ĠOne": 507,
"oug": 508,
"Ġsc": 509,
"Ġfelt": 510,
"Ġtoy": 511
},
"merges": [
[
"h",
"e"
],
[
"Ġ",
"t"
],
[
"Ġ",
"a"
],
[
"Ġ",
"s"
],
[
"Ġ",
"w"
],
[
"n",
"d"
],
[
"Ġt",
"he"
],
[
"e",
"d"
],
[
"Ġa",
"nd"
],
[
"Ġt",
"o"
],
[
"Ġ",
"b"
],
[
"i",
"n"
],
[
"Ġ",
"h"
],
[
"Ġw",
"a"
],
[
"r",
"e"
],
[
"Ġ",
"f"
],
[
"o",
"u"
],
[
"i",
"t"
],
[
"Ġ",
"l"
],
[
"Ġ",
"c"
],
[
"Ġ",
"d"
],
[
"Ġ",
"he"
],
[
"e",
"r"
],
[
"Ġ",
"m"
],
[
"Ġ",
"p"
],
[
"Ġwa",
"s"
],
[
"Ġ",
"T"
],
[
"o",
"m"
],
[
"Ġ",
"o"
],
[
"a",
"y"
],
[
"i",
"s"
],
[
"a",
"r"
],
[
"i",
"l"
],
[
"in",
"g"
],
[
"Ġ",
"g"
],
[
"i",
"d"
],
[
"a",
"t"
],
[
"e",
"n"
],
[
"Ġ",
"n"
],
[
"Ġs",
"a"
],
[
"Ġh",
"a"
],
[
"Ġ",
"S"
],
[
"ĠT",
"he"
],
[
"a",
"n"
],
[
"i",
"m"
],
[
"o",
"r"
],
[
"o",
"n"
],
[
"Ġ",
"it"
],
[
"Ġt",
"h"
],
[
"l",
"l"
],
[
"l",
"e"
],
[
"Ġ",
"H"
],
[
"Ġhe",
"r"
],
[
"e",
"t"
],
[
"o",
"t"
],
[
"i",
"r"
],
[
"ĠS",
"he"
],
[
"ĠH",
"e"
],
[
"e",
"s"
],
[
"Ġ",
"in"
],
[
"o",
"w"
],
[
"u",
"t"
],
[
"v",
"er"
],
[
"ĠThe",
"y"
],
[
"c",
"k"
],
[
"il",
"y"
],
[
"Ġ",
"u"
],
[
"Ġ",
"e"
],
[
"Ġ",
"y"
],
[
"l",
"d"
],
[
"o",
"o"
],
[
"Ġsa",
"id"
],
[
"i",
"g"
],
[
"Ġ",
"\""
],
[
"Ġ",
"r"
],
[
"a",
"m"
],
[
"Ġb",
"e"
],
[
"Ġ",
"I"
],
[
"Ġs",
"t"
],
[
"L",
"ily"
],
[
"it",
"h"
],
[
"k",
"e"
],
[
"c",
"e"
],
[
"p",
"p"
],
[
"v",
"e"
],
[
"Ġw",
"ith"
],
[
"Ġs",
"he"
],
[
"O",
"n"
],
[
"Ġo",
"n"
],
[
"Ġy",
"ou"
],
[
"r",
"i"
],
[
"k",
"ed"
],
[
"Ġp",
"l"
],
[
"Ġh",
"is"
],
[
"Ġo",
"f"
],
[
"n",
"t"
],
[
"Ġs",
"o"
],
[
"a",
"d"
],
[
"Ġd",
"ay"
],
[
"ver",
"y"
],
[
"Ġth",
"at"
],
[
"Ġ",
"Lily"
],
[
"Ġpl",
"ay"
],
[
"Ġu",
"p"
],
[
"Ġw",
"e"
],
[
"Ġm",
"om"
],
[
"Ġha",
"d"
],
[
"Ġthe",
"y"
],
[
"s",
"t"
],
[
"Ġ",
"B"
],
[
"e",
"l"
],
[
"Ġf",
"or"
],
[
"u",
"n"
],
[
"m",
"y"
],
[
"'",
"s"
],
[
"ou",
"ld"
],
[
"Ġl",
"i"
],
[
"T",
"he"
],
[
"Ġha",
"pp"
],
[
"en",
"t"
],
[
"c",
"h"
],
[
"it",
"t"
],
[
"Ġwa",
"nt"
],
[
"he",
"r"
],
[
"Ġs",
"h"
],
[
"ou",
"t"
],
[
"Ġd",
"o"
],
[
"Ġn",
"ot"
],
[
"e",
"nd"
],
[
"om",
"e"
],
[
"a",
"ll"
],
[
"s",
"e"
],
[
"Ġ",
"very"
],
[
"ou",
"nd"
],
[
"l",
"y"
],
[
"im",
"e"
],
[
"itt",
"le"
],
[
"Ġ",
"k"
],
[
"Ġt",
"ime"
],
[
"Ġw",
"h"
],
[
"Ġl",
"ittle"
],
[
"Ġthe",
"re"
],
[
"Ġn",
"e"
],
[
"Ġ",
"is"
],
[
"Ġb",
"ig"
],
[
"h",
"t"
],
[
"Ġhapp",
"y"
],
[
"a",
"l"
],
[
"Ġ",
"M"
],
[
"Ġ",
"re"
],
[
"Ġs",
"m"
],
[
"Ġ",
"Ċ"
],
[
"ri",
"end"
],
[
"Ġf",
"riend"
],
[
"ĠI",
"t"
],
[
"Ġsa",
"w"
],
[
"Ġb",
"ut"
],
[
".",
"\""
],
[
"r",
"a"
],
[
"Ġa",
"n"
],
[
"Ġa",
"s"
],
[
"r",
"y"
],
[
"On",
"e"
],
[
"a",
"ck"
],
[
"id",
"e"
],
[
"a",
"ke"
],
[
"v",
"ed"
],
[
"'",
"t"
],
[
"!",
"\""
],
[
"u",
"g"
],
[
"t",
"er"
],
[
"Ġl",
"o"
],
[
"Ġl",
"oo"
],
[
"On",
"ce"
],
[
"Ġwe",
"re"
],
[
"ĠT",
"im"
],
[
"Ġs",
"e"
],
[
"o",
"re"
],
[
"Ġb",
"o"
],
[
"Ġto",
"o"
],
[
"Ġh",
"im"
],
[
"Ġg",
"o"
],
[
"Ġa",
"re"
],
[
"e",
"c"
],
[
"Ġup",
"on"
],
[
"il",
"l"
],
[
"ir",
"l"
],
[
"Ġwant",
"ed"
],
[
"Ġthe",
"m"
],
[
"ar",
"d"
],
[
"Ġ",
"j"
],
[
"Ġg",
"irl"
],
[
"Ġ",
"out"
],
[
"Ġthe",
"ir"
],
[
"Ġa",
"t"
],
[
"i",
"nd"
],
[
"w",
"ay"
],
[
"u",
"r"
],
[
"f",
"u"
],
[
"Ġd",
"id"
],
[
"Ġ",
"A"
],
[
"Ġs",
"p"
],
[
"Ġsm",
"il"
],
[
"Ġc",
"ould"
],
[
"he",
"n"
],
[
"Ġha",
"ve"
],
[
"a",
"in"
],
[
"ĠB",
"en"
],
[
"Ġw",
"ent"
],
[
"he",
"d"
],
[
"Ġn",
"am"
],
[
"Ġc",
"an"
],
[
"ar",
"t"
],
[
"r",
"om"
],
[
"l",
"p"
],
[
"i",
"c"
],
[
"Ġe",
"x"
],
[
"r",
"ound"
],
[
"Ġfriend",
"s"
],
[
"?",
"\""
],
[
"Ġhe",
"lp"
],
[
"oo",
"d"
],
[
"fu",
"l"
],
[
"Ġk",
"n"
],
[
"Ġ",
"J"
],
[
"Ġa",
"ll"
],
[
"h",
"ing"
],
[
"ig",
"ht"
],
[
"u",
"m"
],
[
"ar",
"k"
],
[
"Ġf",
"un"
],
[
"on",
"e"
],
[
"Ġs",
"ay"
],
[
"Ġb",
"ack"
],
[
"Y",
"ou"
],
[
"Ġnam",
"ed"
],
[
"a",
"re"
],
[
"Ġc",
"l"
],
[
"ĠT",
"om"
],
[
"Ġn",
"o"
],
[
"Ġl",
"e"
],
[
"Ġlo",
"ved"
],
[
"o",
"p"
],
[
"el",
"t"
],
[
"ĠTim",
"my"
],
[
"Ġ",
"One"
],
[
"ou",
"g"
],
[
"Ġs",
"c"
],
[
"Ġf",
"elt"
],
[
"Ġto",
"y"
]
]
}
}