fix

Author: lvdongyi

paddlenlp/transformers/llama/tokenizer.py

@@ -72,9 +72,7 @@ def __init__(
         self.add_bos_token = add_bos_token
         self.add_eos_token = add_eos_token
         self.decode_with_prefix_space = decode_with_prefix_space
-        # self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
-        self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))
-        self.sp_model.Load(vocab_file)
+        self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", True))
 
     @property
     def vocab_size(self):
@@ -101,7 +99,7 @@ def bos_token_id(self) -> Optional[int]:
     def eos_token_id(self) -> Optional[int]:
         return self.sp_model.eos_id()
 
-    def get_spm_processor(self, from_slow=False):
+    def get_spm_processor(self, from_slow=True):
         tokenizer = spm.SentencePieceProcessor(**self.sp_model_kwargs)
         if from_slow:  # no dependency on protobuf
             tokenizer.Load(self.vocab_file)

paddlenlp/transformers/tokenizer_utils_base.py

@@ -1600,7 +1600,10 @@ def _from_pretrained(
         from_hf_hub=False,
         **kwargs,
     ):
-        from_slow = kwargs.get("from_slow", False)
+        if cls.__name__.endswith("Fast"):
+            from_slow = kwargs.get("from_slow", False)
+        else:
+            from_slow = kwargs.get("from_slow", True)
         has_tokenizer_file = resolved_vocab_files.get("tokenizer_file", None) is not None
         if (from_slow or not has_tokenizer_file) and cls.slow_tokenizer_class is not None:
             slow_tokenizer = (cls.slow_tokenizer_class)._from_pretrained(