diff --git a/modelscope/utils/hf_util.py b/modelscope/utils/hf_util.py index 3abcce6d..5722928a 100644 --- a/modelscope/utils/hf_util.py +++ b/modelscope/utils/hf_util.py @@ -91,12 +91,13 @@ def check_hf_code(model_dir: str, auto_class: type, raise FileNotFoundError(f'{config_path} is not found') config_dict = PretrainedConfig.get_config_dict(config_path)[0] auto_class_name = auto_class.__name__ + if auto_class is AutoTokenizerHF: + tokenizer_config = get_tokenizer_config(model_dir) # load from repo if trust_remote_code: has_remote_code = False if auto_class is AutoTokenizerHF: - tokenizer_config_dict = get_tokenizer_config(model_dir) - auto_map = tokenizer_config_dict.get('auto_map', None) + auto_map = tokenizer_config.get('auto_map', None) if auto_map is not None: module_name = auto_map.get(auto_class_name, None) if module_name is not None: @@ -129,7 +130,10 @@ def check_hf_code(model_dir: str, auto_class: type, f'{model_type} not found in HF `CONFIG_MAPPING`{trust_remote_code_info}' ) elif auto_class is AutoTokenizerHF: - if model_type not in TOKENIZER_MAPPING_NAMES: + tokenizer_class = tokenizer_config.get('tokenizer_class') + if tokenizer_class is not None: + return + if model_type in TOKENIZER_MAPPING_NAMES: raise ValueError( f'{model_type} not found in HF `TOKENIZER_MAPPING_NAMES`{trust_remote_code_info}' )