| { | |
| "version": "1.0", | |
| "truncation": null, | |
| "padding": null, | |
| "added_tokens": [ | |
| { | |
| "id": 0, | |
| "content": "<pad>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 1, | |
| "content": "<unk>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 2, | |
| "content": "<s>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 3, | |
| "content": "</s>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 4, | |
| "content": "<mask>", | |
| "single_word": false, | |
| "lstrip": true, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": true | |
| }, | |
| { | |
| "id": 5, | |
| "content": " ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 6, | |
| "content": "!", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 7, | |
| "content": "\"", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 8, | |
| "content": "$", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 9, | |
| "content": "%", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 10, | |
| "content": "'", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 11, | |
| "content": "(", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 12, | |
| "content": ")", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 13, | |
| "content": ",", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 14, | |
| "content": "-", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 15, | |
| "content": ".", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 16, | |
| "content": "0", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 17, | |
| "content": "1", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 18, | |
| "content": "2", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 19, | |
| "content": "3", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 20, | |
| "content": "4", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 21, | |
| "content": "5", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 22, | |
| "content": "6", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 23, | |
| "content": "7", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 24, | |
| "content": "8", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 25, | |
| "content": "9", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 26, | |
| "content": ":", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 27, | |
| "content": ";", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 28, | |
| "content": "?", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 29, | |
| "content": "H", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 30, | |
| "content": "I", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 31, | |
| "content": "N", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 32, | |
| "content": "a", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 33, | |
| "content": "b", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 34, | |
| "content": "d", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 35, | |
| "content": "g", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 36, | |
| "content": "i", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 37, | |
| "content": "j", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 38, | |
| "content": "n", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 39, | |
| "content": "o", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 40, | |
| "content": "p", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 41, | |
| "content": "s", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 42, | |
| "content": "t", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 43, | |
| "content": " ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 44, | |
| "content": "°", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 45, | |
| "content": "²", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 46, | |
| "content": "–", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 47, | |
| "content": "—", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 48, | |
| "content": "ⴰ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 49, | |
| "content": "ⴱ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 50, | |
| "content": "ⴳ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 51, | |
| "content": "ⴷ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 52, | |
| "content": "ⴹ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 53, | |
| "content": "ⴻ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 54, | |
| "content": "ⴼ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 55, | |
| "content": "ⴽ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 56, | |
| "content": "ⵀ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 57, | |
| "content": "ⵃ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 58, | |
| "content": "ⵄ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 59, | |
| "content": "ⵅ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 60, | |
| "content": "ⵇ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 61, | |
| "content": "ⵉ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 62, | |
| "content": "ⵊ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 63, | |
| "content": "ⵍ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 64, | |
| "content": "ⵎ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 65, | |
| "content": "ⵏ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 66, | |
| "content": "ⵒ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 67, | |
| "content": "ⵓ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 68, | |
| "content": "ⵔ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 69, | |
| "content": "ⵕ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 70, | |
| "content": "ⵖ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 71, | |
| "content": "ⵙ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 72, | |
| "content": "ⵚ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 73, | |
| "content": "ⵛ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 74, | |
| "content": "ⵜ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 75, | |
| "content": "ⵟ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 76, | |
| "content": "ⵡ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 77, | |
| "content": "ⵢ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 78, | |
| "content": "ⵣ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 79, | |
| "content": "ⵥ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 80, | |
| "content": "ⵯ", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| }, | |
| { | |
| "id": 81, | |
| "content": "⵰", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": true, | |
| "special": false | |
| } | |
| ], | |
| "normalizer": null, | |
| "pre_tokenizer": { | |
| "type": "ByteLevel", | |
| "add_prefix_space": false, | |
| "trim_offsets": true, | |
| "use_regex": true | |
| }, | |
| "post_processor": { | |
| "type": "RobertaProcessing", | |
| "sep": [ | |
| "</s>", | |
| 3 | |
| ], | |
| "cls": [ | |
| "<s>", | |
| 2 | |
| ], | |
| "trim_offsets": true, | |
| "add_prefix_space": false | |
| }, | |
| "decoder": { | |
| "type": "ByteLevel", | |
| "add_prefix_space": true, | |
| "trim_offsets": true, | |
| "use_regex": true | |
| }, | |
| "model": { | |
| "type": "BPE", | |
| "dropout": null, | |
| "unk_token": null, | |
| "continuing_subword_prefix": "", | |
| "end_of_word_suffix": "", | |
| "fuse_unk": false, | |
| "byte_fallback": false, | |
| "ignore_merges": false, | |
| "vocab": { | |
| "<pad>": 0, | |
| "<unk>": 1, | |
| "<s>": 2, | |
| "</s>": 3, | |
| "<mask>": 4 | |
| }, | |
| "merges": [] | |
| } | |
| } |