diff --git a/modelscope/utils/hf_util/patcher.py b/modelscope/utils/hf_util/patcher.py
index ecc33662..02afb758 100644
--- a/modelscope/utils/hf_util/patcher.py
+++ b/modelscope/utils/hf_util/patcher.py
@@ -321,9 +321,8 @@ def _patch_pretrained_class(all_imported_modules, wrap=False):
         if var is None or not hasattr(var, '__name__'):
             continue
         name = var.__name__
-        need_model = 'model' in name.lower() or 'processor' in name.lower(
-        ) or 'extractor' in name.lower() or 'pipeline' in name.lower()
-        if need_model:
+        skip_model = 'tokenizer' in name.lower() or 'config' in name.lower()
+        if not skip_model:
             ignore_file_pattern_kwargs = {}
         else:
             ignore_file_pattern_kwargs = {