Add fast tokenizer for BARTpho

docs/source/en/model_doc/bartpho.mdx

@@ -70,13 +70,14 @@ Tips:
 >>> print(tokenizer.decode(predictions).split())
 ```
 
-- This implementation is only for tokenization: "monolingual_vocab_file" consists of Vietnamese-specialized types
-  extracted from the pre-trained SentencePiece model "vocab_file" that is available from the multilingual XLM-RoBERTa.
-  Other languages, if employing this pre-trained multilingual SentencePiece model "vocab_file" for subword
-  segmentation, can reuse BartphoTokenizer with their own language-specialized "monolingual_vocab_file".
+- This implementation is only for tokenization.
 
 This model was contributed by [dqnguyen](https://huggingface.co/dqnguyen). The original code can be found [here](https://github.com/VinAIResearch/BARTpho).
 
 ## BartphoTokenizer
 
 [[autodoc]] BartphoTokenizer
+
+## BartphoTokenizerFast
+
+[[autodoc]] BartphoTokenizerFast

src/transformers/__init__.py

@@ -478,6 +478,7 @@
     _import_structure["models.albert"].append("AlbertTokenizerFast")
     _import_structure["models.bart"].append("BartTokenizerFast")
     _import_structure["models.barthez"].append("BarthezTokenizerFast")
+    _import_structure["models.bartpho"].append("BartphoTokenizerFast")
     _import_structure["models.bert"].append("BertTokenizerFast")
     _import_structure["models.big_bird"].append("BigBirdTokenizerFast")
     _import_structure["models.blenderbot"].append("BlenderbotTokenizerFast")
@@ -2959,6 +2960,7 @@
         from .models.albert import AlbertTokenizerFast
         from .models.bart import BartTokenizerFast
         from .models.barthez import BarthezTokenizerFast
+        from .models.bartpho import BartphoTokenizerFast
         from .models.bert import BertTokenizerFast
         from .models.big_bird import BigBirdTokenizerFast
         from .models.blenderbot import BlenderbotTokenizerFast

src/transformers/convert_slow_tokenizer.py

@@ -551,6 +551,60 @@ def post_processor(self):
         )
 
 
+class BartphoConverter(SpmConverter):
+    def vocab(self, proto):
+        vocab = [
+            ("<s>", 0.0),
+            ("<pad>", 0.0),
+            ("</s>", 0.0),
+            ("<unk>", 0.0),
+        ]
+        vocab += [(piece.piece, piece.score) for piece in proto.pieces[3:]]
+        vocab += [
+            ("ar_AR", 0.0),
+            ("cs_CZ", 0.0),
+            ("de_DE", 0.0),
+            ("en_XX", 0.0),
+            ("es_XX", 0.0),
+            ("et_EE", 0.0),
+            ("fi_FI", 0.0),
+            ("fr_XX", 0.0),
+            ("gu_IN", 0.0),
+            ("hi_IN", 0.0),
+            ("it_IT", 0.0),
+            ("ja_XX", 0.0),
+            ("kk_KZ", 0.0),
+            ("ko_KR", 0.0),
+            ("lt_LT", 0.0),
+            ("lv_LV", 0.0),
+            ("my_MM", 0.0),
+            ("ne_NP", 0.0),
+            ("nl_XX", 0.0),
+            ("ro_RO", 0.0),
+            ("ru_RU", 0.0),
+            ("si_LK", 0.0),
+            ("tr_TR", 0.0),
+            ("vi_VN", 0.0),
+            ("zh_CN", 0.0),
+        ]
+        vocab += [("<mask>", 0.0)]
+        return vocab
+
+    def unk_id(self, proto):
+        unk_id = 3
+        return unk_id
+
+    def post_processor(self):
+        return processors.TemplateProcessing(
+            single="<s> $A </s>",
+            pair="<s> $A </s> </s> $B </s>",
+            special_tokens=[
+                ("<s>", self.original_tokenizer.convert_tokens_to_ids("<s>")),
+                ("</s>", self.original_tokenizer.convert_tokens_to_ids("</s>")),
+            ],
+        )
+
+
 class CamembertConverter(SpmConverter):
     def vocab(self, proto):
         vocab = [
@@ -1004,6 +1058,7 @@ def post_processor(self):
     "AlbertTokenizer": AlbertConverter,
     "BartTokenizer": RobertaConverter,
     "BarthezTokenizer": BarthezConverter,
+    "BartphoTokenizer": BartphoConverter,
     "BertTokenizer": BertConverter,
     "BigBirdTokenizer": BigBirdConverter,
     "BlenderbotTokenizer": BlenderbotConverter,

src/transformers/models/auto/tokenization_auto.py

@@ -61,7 +61,13 @@
                     "BarthezTokenizerFast" if is_tokenizers_available() else None,
                 ),
             ),
-            ("bartpho", ("BartphoTokenizer", None)),
+            (
+                "bartpho",
+                (
+                    "BartphoTokenizer" if is_sentencepiece_available() else None,
+                    "BartphoTokenizerFast" if is_tokenizers_available() else None,
+                ),
+            ),
             ("bert", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)),
             ("bert-generation", ("BertGenerationTokenizer" if is_sentencepiece_available() else None, None)),
             ("bert-japanese", ("BertJapaneseTokenizer", None)),

src/transformers/models/bartpho/__init__.py

@@ -18,7 +18,7 @@
 
 from typing import TYPE_CHECKING
 
-from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available
+from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available, is_tokenizers_available
 
 
 _import_structure = {}
@@ -31,6 +31,14 @@
 else:
     _import_structure["tokenization_bartpho"] = ["BartphoTokenizer"]
 
+try:
+    if not is_tokenizers_available():
+        raise OptionalDependencyNotAvailable()
+except OptionalDependencyNotAvailable:
+    pass
+else:
+    _import_structure["tokenization_bartpho_fast"] = ["BartphoTokenizerFast"]
+
 if TYPE_CHECKING:
     try:
         if not is_sentencepiece_available():
@@ -40,6 +48,14 @@
     else:
         from .tokenization_bartpho import BartphoTokenizer
 
+    try:
+        if not is_tokenizers_available():
+            raise OptionalDependencyNotAvailable()
+    except OptionalDependencyNotAvailable:
+        pass
+    else:
+        from .tokenization_bartpho_fast import BartphoTokenizerFast
+
 else:
     import sys
 

src/transformers/models/bartpho/tokenization_bartpho.py

@@ -29,19 +29,20 @@
 
 SPIECE_UNDERLINE = "▁"
 
-VOCAB_FILES_NAMES = {"vocab_file": "sentencepiece.bpe.model", "monolingual_vocab_file": "dict.txt"}
+VOCAB_FILES_NAMES = {"vocab_file": "sentencepiece.bpe.model"}
 
 PRETRAINED_VOCAB_FILES_MAP = {
     "vocab_file": {
         "vinai/bartpho-syllable": "https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model",
     },
-    "monolingual_vocab_file": {
-        "vinai/bartpho-syllable": "https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt",
-    },
 }
 
 PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"vinai/bartpho-syllable": 1024}
 
+# fmt: off
+FAIRSEQ_LANGUAGE_CODES = ["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN"]
+# fmt: on
+
 
 class BartphoTokenizer(PreTrainedTokenizer):
     """
@@ -52,11 +53,7 @@ class BartphoTokenizer(PreTrainedTokenizer):
 
     Args:
         vocab_file (`str`):
-            Path to the vocabulary file. This vocabulary is the pre-trained SentencePiece model available from the
-            multilingual XLM-RoBERTa, also used in mBART, consisting of 250K types.
-        monolingual_vocab_file (`str`):
-            Path to the monolingual vocabulary file. This monolingual vocabulary consists of Vietnamese-specialized
-            types extracted from the multilingual vocabulary vocab_file of 250K types.
+            Path to the vocabulary file.
         bos_token (`str`, *optional*, defaults to `"<s>"`):
             The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.
 
@@ -123,7 +120,6 @@ class BartphoTokenizer(PreTrainedTokenizer):
     def __init__(
         self,
         vocab_file,
-        monolingual_vocab_file,
         bos_token="<s>",
         eos_token="</s>",
         sep_token="</s>",
@@ -151,27 +147,30 @@ def __init__(
             **kwargs,
         )
 
-        self.vocab_file = vocab_file
-        self.monolingual_vocab_file = monolingual_vocab_file
         self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
         self.sp_model.Load(str(vocab_file))
+        self.vocab_file = vocab_file
+
+        # Original fairseq vocab and spm vocab must be "aligned":
+        # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
+        # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
+        # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
+        # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'
 
-        # Load the reduced vocab
-
-        # Keep order of special tokens for backward compatibility
-        self.fairseq_tokens_to_ids = {}
-        cnt = 0
-        for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
-            if str(token) not in self.fairseq_tokens_to_ids:
-                self.fairseq_tokens_to_ids[str(token)] = cnt
-                cnt += 1
-        with open(monolingual_vocab_file, "r", encoding="utf-8") as f:
-            for line in f.readlines():
-                token = line.strip().split()[0]
-                self.fairseq_tokens_to_ids[token] = len(self.fairseq_tokens_to_ids)
-        if str(mask_token) not in self.fairseq_tokens_to_ids:
-            self.fairseq_tokens_to_ids[str(mask_token)] = len(self.fairseq_tokens_to_ids)
+        # Mimic fairseq token-to-id alignment for the first 4 token
+        self.fairseq_tokens_to_ids = {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}
 
+        # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
+        self.fairseq_offset = 1
+
+        self.sp_model_size = len(self.sp_model)
+        self.lang_code_to_id = {
+            code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(FAIRSEQ_LANGUAGE_CODES)
+        }
+        self.id_to_lang_code = {v: k for k, v in self.lang_code_to_id.items()}
+        self.fairseq_tokens_to_ids["<mask>"] = len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset
+
+        self.fairseq_tokens_to_ids.update(self.lang_code_to_id)
         self.fairseq_ids_to_tokens = {v: k for k, v in self.fairseq_tokens_to_ids.items()}
 
     def __getstate__(self):
@@ -195,7 +194,7 @@ def build_inputs_with_special_tokens(
     ) -> List[int]:
         """
         Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
-        adding special tokens. An BARTPho sequence has the following format:
+        adding special tokens. A BARTpho sequence has the following format:
 
         - single sequence: `<s> X </s>`
         - pair of sequences: `<s> A </s></s> B </s>`
@@ -248,7 +247,7 @@ def create_token_type_ids_from_sequences(
         self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
     ) -> List[int]:
         """
-        Create a mask from the two sequences passed to be used in a sequence-pair classification task. BARTPho does not
+        Create a mask from the two sequences passed to be used in a sequence-pair classification task. BARTpho does not
         make use of token type ids, therefore a list of zeros is returned.
 
         Args:
@@ -271,7 +270,7 @@ def create_token_type_ids_from_sequences(
 
     @property
     def vocab_size(self):
-        return len(self.fairseq_ids_to_tokens)
+        return len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset + 1  # Plus 1 for the mask token
 
     def get_vocab(self):
         vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
@@ -285,12 +284,16 @@ def _convert_token_to_id(self, token):
         """Converts a token (str) in an id using the vocab."""
         if token in self.fairseq_tokens_to_ids:
             return self.fairseq_tokens_to_ids[token]
-        else:
-            return self.unk_token_id
+        spm_id = self.sp_model.PieceToId(token)
+
+        # Need to return unknown token if the SP model returned 0
+        return spm_id + self.fairseq_offset if spm_id else self.unk_token_id
 
     def _convert_id_to_token(self, index):
         """Converts an index (integer) in a token (str) using the vocab."""
-        return self.fairseq_ids_to_tokens[index]
+        if index in self.fairseq_ids_to_tokens:
+            return self.fairseq_ids_to_tokens[index]
+        return self.sp_model.IdToPiece(index - self.fairseq_offset)
 
     def convert_tokens_to_string(self, tokens):
         """Converts a sequence of tokens (strings for sub-words) in a single string."""
@@ -304,10 +307,6 @@ def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] =
         out_vocab_file = os.path.join(
             save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
         )
-        out_monolingual_vocab_file = os.path.join(
-            save_directory,
-            (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["monolingual_vocab_file"],
-        )
 
         if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
             copyfile(self.vocab_file, out_vocab_file)
@@ -316,14 +315,4 @@ def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] =
                 content_spiece_model = self.sp_model.serialized_model_proto()
                 fi.write(content_spiece_model)
 
-        if os.path.abspath(self.monolingual_vocab_file) != os.path.abspath(
-            out_monolingual_vocab_file
-        ) and os.path.isfile(self.monolingual_vocab_file):
-            copyfile(self.monolingual_vocab_file, out_monolingual_vocab_file)
-        elif not os.path.isfile(self.monolingual_vocab_file):
-            with open(out_monolingual_vocab_file, "w", encoding="utf-8") as fp:
-                for token in self.fairseq_tokens_to_ids:
-                    if token not in self.all_special_tokens:
-                        fp.write(f"{str(token)} \n")
-
-        return out_vocab_file, out_monolingual_vocab_file
+        return (out_vocab_file,)