rm redundent comment, resolve complicate

lvdongyi · lvdongyi · commit 2ceeccb25a47 · 2024-10-23T17:44:04.000+08:00
diff --git a/paddlenlp/transformers/tokenizer_utils_base.py b/paddlenlp/transformers/tokenizer_utils_base.py
@@ -1531,8 +1531,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
             "added_tokens_file": ADDED_TOKENS_FILE,
             "special_tokens_map_file": SPECIAL_TOKENS_MAP_FILE,
             "tokenizer_config_file": TOKENIZER_CONFIG_FILE,
-            "chat_template_file": CHAT_TEMPLATE_CONFIG_NAME,  # what's this
-            # "tokenizer_file": FULL_TOKENIZER_FILE,
+            "chat_template_file": CHAT_TEMPLATE_CONFIG_NAME,
         }
 
         vocab_files_target = {**cls.resource_files_names, **additional_files_names}

Original file line number	Diff line number	Diff line change
`@@ -1531,8 +1531,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, args, *kwargs):`
`1531`	`1531`	`"added_tokens_file": ADDED_TOKENS_FILE,`
`1532`	`1532`	`"special_tokens_map_file": SPECIAL_TOKENS_MAP_FILE,`
`1533`	`1533`	`"tokenizer_config_file": TOKENIZER_CONFIG_FILE,`
`1534`		`- "chat_template_file": CHAT_TEMPLATE_CONFIG_NAME, # what's this`
`1535`		`- # "tokenizer_file": FULL_TOKENIZER_FILE,`
	`1534`	`+ "chat_template_file": CHAT_TEMPLATE_CONFIG_NAME,`
`1536`	`1535`	`}`
`1537`	`1536`
`1538`	`1537`	`vocab_files_target = {cls.resource_files_names, additional_files_names}`