fix

Author: lvdongyi

paddlenlp/transformers/llama/tokenizer.py

@@ -72,9 +72,7 @@ def __init__(
         self.add_bos_token = add_bos_token
         self.add_eos_token = add_eos_token
         self.decode_with_prefix_space = decode_with_prefix_space
-        # self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
         self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))
-        self.sp_model.Load(vocab_file)
 
     @property
     def vocab_size(self):

paddlenlp/transformers/tokenizer_utils_base.py

@@ -1600,7 +1600,10 @@ def _from_pretrained(
         from_hf_hub=False,
         **kwargs,
     ):
-        from_slow = kwargs.get("from_slow", False)
+        if cls.__name__.endswith("Fast"):
+            from_slow = kwargs.get("from_slow", False)
+        else:
+            from_slow = kwargs.get("from_slow", True)
         has_tokenizer_file = resolved_vocab_files.get("tokenizer_file", None) is not None
         if (from_slow or not has_tokenizer_file) and cls.slow_tokenizer_class is not None:
             slow_tokenizer = (cls.slow_tokenizer_class)._from_pretrained(