tokenizer.json

{
  "version": "1.0",
  "truncation": null,
  "padding": null,
  "added_tokens": [
    {
      "id": 0,
      "content": "[PAD]",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 1,
      "content": "[BOS]",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 2,
      "content": "[EOS]",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    },
    {
      "id": 3,
      "content": "[UNK]",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": false,
      "special": true
    }
  ],
  "normalizer": null,
  "pre_tokenizer": {
    "type": "WhitespaceSplit"
  },
  "post_processor": {
    "type": "TemplateProcessing",
    "single": [
      {
        "Sequence": {
          "id": "A",
          "type_id": 0
        }
      }
    ],
    "pair": [
      {
        "Sequence": {
          "id": "A",
          "type_id": 0
        }
      },
      {
        "Sequence": {
          "id": "B",
          "type_id": 1
        }
      }
    ],
    "special_tokens": {}
  },
  "decoder": null,
  "model": {
    "type": "WordLevel",
    "vocab": {
      "[PAD]": 0,
      "[BOS]": 1,
      "[EOS]": 2,
      "[UNK]": 3,
      "[ROW_00]": 4,
      "[ROW_01]": 5,
      "[ROW_02]": 6,
      "[ROW_03]": 7,
      "[ROW_04]": 8,
      "[ROW_05]": 9,
      "[ROW_06]": 10,
      "[ROW_07]": 11,
      "[ROW_08]": 12,
      "[ROW_09]": 13,
      "[ROW_10]": 14,
      "[ROW_11]": 15,
      "[ROW_12]": 16,
      "[ROW_13]": 17,
      "[ROW_14]": 18,
      "[ROW_15]": 19,
      "[ROW_16]": 20,
      "[ROW_17]": 21,
      "[ROW_18]": 22,
      "[ROW_19]": 23,
      "[ROW_20]": 24,
      "[ROW_21]": 25,
      "[ROW_22]": 26,
      "[ROW_23]": 27,
      "[ROW_24]": 28,
      "[ROW_25]": 29,
      "[ROW_26]": 30,
      "[ROW_27]": 31,
      "[ROW_28]": 32,
      "[ROW_29]": 33,
      "[ROW_30]": 34,
      "[ROW_31]": 35,
      "[ROW_32]": 36,
      "[ROW_33]": 37,
      "[ROW_34]": 38,
      "[ROW_35]": 39,
      "[ROW_36]": 40,
      "[ROW_37]": 41,
      "[ROW_38]": 42,
      "[ROW_39]": 43,
      "[ROW_40]": 44,
      "[ROW_41]": 45,
      "[ROW_42]": 46,
      "[ROW_43]": 47,
      "[ROW_44]": 48,
      "[ROW_45]": 49,
      "[ROW_46]": 50,
      "[ROW_47]": 51,
      "[ROW_48]": 52,
      "[ROW_49]": 53,
      "[ROW_50]": 54,
      "[ROW_51]": 55,
      "[ROW_52]": 56,
      "[ROW_53]": 57,
      "[ROW_54]": 58,
      "[ROW_55]": 59,
      "[ROW_56]": 60,
      "[ROW_57]": 61,
      "[ROW_58]": 62,
      "[ROW_59]": 63,
      "[ROW_60]": 64,
      "[ROW_61]": 65,
      "[ROW_62]": 66,
      "[ROW_63]": 67,
      ";": 68,
      "<": 69,
      "=": 70,
      ">": 71,
      "?": 72,
      "@": 73,
      "A": 74,
      "B": 75,
      "C": 76,
      "D": 77,
      "E": 78,
      "F": 79,
      "G": 80,
      "H": 81,
      "I": 82,
      "J": 83,
      "K": 84,
      "L": 85,
      "M": 86,
      "N": 87,
      "O": 88,
      "P": 89,
      "Q": 90,
      "R": 91,
      "S": 92,
      "T": 93,
      "U": 94,
      "V": 95,
      "W": 96,
      "X": 97,
      "Y": 98,
      "Z": 99,
      "[": 100,
      "\\": 101,
      "]": 102,
      "^": 103,
      "_": 104,
      "`": 105,
      "a": 106,
      "b": 107,
      "c": 108,
      "d": 109,
      "e": 110,
      "f": 111,
      "g": 112,
      "h": 113,
      "i": 114,
      "j": 115,
      "k": 116,
      "l": 117,
      "m": 118,
      "n": 119,
      "o": 120,
      "p": 121,
      "q": 122,
      "r": 123,
      "s": 124,
      "t": 125,
      "u": 126,
      "v": 127,
      "w": 128,
      "x": 129,
      "y": 130,
      "z": 131,
      "~": 132
    },
    "unk_token": "[UNK]"
  }
}