Adapt tokenization_interns1.py to transformers>=5.0.0

tokenization_interns1.py

@@ -25,24 +25,27 @@ import regex as re
 import sentencepiece as spm
 from collections import OrderedDict
 
-from transformers.tokenization_utils import PreTrainedTokenizer
 from transformers.tokenization_utils_base import AddedToken, TextInput
-from transformers.models.qwen2.tokenization_qwen2 import Qwen2Tokenizer
 from transformers.utils import logging
+import transformers
+from packaging import version
+if version.parse(transformers.__version__) >= version.parse("5.0.0"):
+    from transformers.tokenization_python import PreTrainedTokenizer
+else:
+    from transformers.tokenization_utils import PreTrainedTokenizer
 
 
 logger = logging.get_logger(__name__)
 
 try:
-    from rdkit import Chem
-    from rdkit import RDLogger
+    from rdkit import Chem, RDLogger
 
     RDLogger.DisableLog("rdApp.error")
     RDLogger.DisableLog("rdApp.*")
     RDKIT_AVAILABLE = True
 except ImportError:
     logger.warning_once(
-        f"If tokenization with SMILES formula is of necessity, please 'pip install RDKit' for better tokenization quality."
+        "If tokenization with SMILES formula is of necessity, please 'pip install RDKit' for better tokenization quality."
     )
     RDKIT_AVAILABLE = False
 
@@ -343,7 +346,48 @@ class SmilesCheckModule(InternS1CheckModuleMixin):
         return self.check_brackets(text)
 
 
-class InternS1Tokenizer(Qwen2Tokenizer):
+@lru_cache
+# Copied from transformers.models.gpt2.tokenization_gpt2.bytes_to_unicode
+def bytes_to_unicode():
+    """
+    Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
+    characters the bpe code barfs on.
+
+    The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
+    if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
+    decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
+    tables between utf-8 bytes and unicode strings.
+    """
+    bs = (
+        list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
+    )
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8 + n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+
+# Copied from transformers.models.gpt2.tokenization_gpt2.get_pairs
+def get_pairs(word):
+    """
+    Return set of symbol pairs in a word.
+
+    Word is represented as tuple of symbols (symbols being variable-length strings).
+    """
+    pairs = set()
+    prev_char = word[0]
+    for char in word[1:]:
+        pairs.add((prev_char, char))
+        prev_char = char
+    return pairs
+
+
+class InternS1Tokenizer(PreTrainedTokenizer):
     """
     Construct an InternS1 tokenizer. Based on byte-level Byte-Pair-Encoding.
 
@@ -408,6 +452,54 @@ class InternS1Tokenizer(Qwen2Tokenizer):
         split_special_tokens=False,
         **kwargs,
     ):
+        bos_token = (
+            AddedToken(bos_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(bos_token, str)
+            else bos_token
+        )
+        eos_token = (
+            AddedToken(eos_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(eos_token, str)
+            else eos_token
+        )
+        unk_token = (
+            AddedToken(unk_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(unk_token, str)
+            else unk_token
+        )
+        pad_token = (
+            AddedToken(pad_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(pad_token, str)
+            else pad_token
+        )
+
+        with open(vocab_file, encoding="utf-8") as vocab_handle:
+            self.encoder = json.load(vocab_handle)
+        self.decoder = {v: k for k, v in self.encoder.items()}
+        self.errors = errors  # how to handle errors in decoding
+        self.byte_encoder = bytes_to_unicode()
+        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
+        bpe_merges = []
+        with open(merges_file, encoding="utf-8") as merges_handle:
+            for i, line in enumerate(merges_handle):
+                line = line.strip()
+                if (i == 0 and line.startswith("#version:")) or not line:
+                    continue
+                bpe_merges.append(tuple(line.split()))
+        self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
+        # NOTE: the cache can grow without bound and will get really large for long running processes
+        # (esp. for texts of language that do not use space between word, e.g. Chinese); technically
+        # not a memory leak but appears as one.
+        # GPT2Tokenizer has the same problem, so let's be consistent.
+        self.cache = {}
+
+        self.pat = re.compile(PRETOKENIZE_REGEX)
+
+        if kwargs.get("add_prefix_space", False):
+            logger.warning_once(
+                f"{self.__class__.__name} does not support `add_prefix_space`, setting it to True has no effect."
+            )
+
         self.extra_tokenizer_start_mapping = {}
         self.extra_tokenizer_end_mapping = {}
         self._extra_special_tokens = []
@@ -460,6 +552,7 @@ class InternS1Tokenizer(Qwen2Tokenizer):
             pad_token=pad_token,
             clean_up_tokenization_spaces=clean_up_tokenization_spaces,
             split_special_tokens=split_special_tokens,
+            special_tokens_pattern="none",
             **kwargs,
         )
 
@@ -497,6 +590,10 @@ class InternS1Tokenizer(Qwen2Tokenizer):
         """Overload method"""
         return self.vocab_size
 
+    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.get_vocab
+    def get_vocab(self):
+        return dict(self.encoder, **self.added_tokens_encoder)
+
     @property
     def logical_auto_tokens(self):
         """Tokens that won't be decoded and only for switching tokenizer"""
@@ -633,9 +730,6 @@ class InternS1Tokenizer(Qwen2Tokenizer):
 
         text, kwargs = self.prepare_for_tokenization(text, **kwargs)
 
-        if kwargs:
-            logger.warning(f"Keyword arguments {kwargs} not recognized.")
-
         if hasattr(self, "do_lower_case") and self.do_lower_case:
             # convert non-special tokens to lowercase. Might be super slow as well?
             escaped_special_toks = [re.escape(s_tok) for s_tok in (self.all_special_tokens)]
@@ -785,6 +879,7 @@ class InternS1Tokenizer(Qwen2Tokenizer):
             self._added_tokens_encoder[token.content] = token_index
             if self.verbose:
                 logger.info(f"Adding {token} to the vocabulary")
+
         self._update_trie()
         self._update_total_vocab_size()
 
@@ -814,6 +909,49 @@ class InternS1Tokenizer(Qwen2Tokenizer):
         else:
             return self._bpe_tokenize(text)
 
+    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.bpe
+    def bpe(self, token):
+        if token in self.cache:
+            return self.cache[token]
+        word = tuple(token)
+        pairs = get_pairs(word)
+
+        if not pairs:
+            return token
+
+        while True:
+            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
+            if bigram not in self.bpe_ranks:
+                break
+            first, second = bigram
+            new_word = []
+            i = 0
+            while i < len(word):
+                try:
+                    j = word.index(first, i)
+                except ValueError:
+                    new_word.extend(word[i:])
+                    break
+                else:
+                    new_word.extend(word[i:j])
+                    i = j
+
+                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
+                    new_word.append(first + second)
+                    i += 2
+                else:
+                    new_word.append(word[i])
+                    i += 1
+            new_word = tuple(new_word)
+            word = new_word
+            if len(word) == 1:
+                break
+            else:
+                pairs = get_pairs(word)
+        word = " ".join(word)
+        self.cache[token] = word
+        return word
+
     def _bpe_tokenize(self, text, **kwargs):
         text = text.replace(
             "▁", " "