Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2025-12-16 16:27:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix processing ignore files (#1398)

Browse Source
This commit is contained in:
tastelikefeet
2025-07-07 14:22:33 +08:00
committed by GitHub
parent f4217e6183
commit 6c619155ac
1 changed files with 2 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
modelscope/utils/hf_util/patcher.py
View File

@@ -239,24 +239,14 @@ def _patch_pretrained_class(all_imported_modules, wrap=False):
from transformers import CONFIG_NAME
extra_allow_file_pattern = [CONFIG_NAME, r'*.py']
elif 'Tokenizer' in module_class.__name__:
from transformers.tokenization_utils import ADDED_TOKENS_FILE
from transformers.tokenization_utils import SPECIAL_TOKENS_MAP_FILE
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
from transformers.tokenization_utils_base import FULL_TOKENIZER_FILE
extra_allow_file_pattern = list(
(cls.vocab_files_names.values()) if cls is not None
and hasattr(cls, 'vocab_files_names') else []) + [
ADDED_TOKENS_FILE, SPECIAL_TOKENS_MAP_FILE,
TOKENIZER_CONFIG_FILE, FULL_TOKENIZER_FILE,
'chat_template.jinja', r'*.py'
'chat_template.jinja', r'*.json', r'*.py'
] # noqa
elif 'Processor' in module_class.__name__:
from transformers.utils import FEATURE_EXTRACTOR_NAME
from transformers.utils import PROCESSOR_NAME
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
extra_allow_file_pattern = [
FEATURE_EXTRACTOR_NAME, TOKENIZER_CONFIG_FILE,
PROCESSOR_NAME, r'*.py'
'chat_template.jinja', r'*.json', r'*.py'
]
kwargs['allow_file_pattern'] = extra_allow_file_pattern