@@ -93,7 +93,7 @@ def __init__(
9393 pad_token = AddedToken (pad_token , lstrip = False , rstrip = False ) if isinstance (pad_token , str ) else pad_token
9494 eos_token = AddedToken (eos_token , lstrip = False , rstrip = False ) if isinstance (eos_token , str ) else eos_token
9595 unk_token = AddedToken (unk_token , lstrip = False , rstrip = False ) if isinstance (unk_token , str ) else unk_token
96- bos_token = AddedToken (unk_token , lstrip = False , rstrip = False ) if isinstance (bos_token , str ) else bos_token
96+ bos_token = AddedToken (bos_token , lstrip = False , rstrip = False ) if isinstance (bos_token , str ) else bos_token
9797
9898 self ._build_special_tokens_map_extended (
9999 bos_token = bos_token ,
@@ -105,7 +105,10 @@ def __init__(
105105 value = kwargs .pop ("added_tokens_decoder" , {})
106106 additional_special_tokens = []
107107 for _ , token_kwargs in value .items ():
108- content = AddedToken (** token_kwargs )
108+ if isinstance (token_kwargs , AddedToken ):
109+ content = token_kwargs
110+ else :
111+ content = AddedToken (** token_kwargs )
109112 additional_special_tokens .append (content )
110113 if len (additional_special_tokens ) > 0 :
111114 self ._build_special_tokens_map_extended (
0 commit comments