From ad3a45bc3946736200248ba71ef6cc7fb66a51cb Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Thu, 6 Feb 2025 13:54:29 +0800 Subject: [PATCH 01/34] bump version --- modelscope/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modelscope/version.py b/modelscope/version.py index 031a86b4..8984216b 100644 --- a/modelscope/version.py +++ b/modelscope/version.py @@ -1,5 +1,5 @@ # Make sure to modify __release_datetime__ to release time when making official release. -__version__ = '2.0.0' +__version__ = '1.23.0' # default release datetime for branches under active development is set # to be a time far-far-away-into-the-future -__release_datetime__ = '2099-09-06 00:00:00' +__release_datetime__ = '2025-02-10 00:00:00' From f3985d9b102e0563843dd977cae611a988613e80 Mon Sep 17 00:00:00 2001 From: Yingda Chen Date: Thu, 6 Feb 2025 17:05:32 +0800 Subject: [PATCH 02/34] fix cache path (#1211) Co-authored-by: Yingda Chen (cherry picked from commit 369606cda8920a31f23f58c722428a78887e5ccc) --- modelscope/hub/snapshot_download.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modelscope/hub/snapshot_download.py b/modelscope/hub/snapshot_download.py index 35b0f3a4..77b49847 100644 --- a/modelscope/hub/snapshot_download.py +++ b/modelscope/hub/snapshot_download.py @@ -20,7 +20,7 @@ from modelscope.utils.constant import (DEFAULT_DATASET_REVISION, DEFAULT_REPOSITORY_REVISION, REPO_TYPE_DATASET, REPO_TYPE_MODEL, REPO_TYPE_SUPPORT) -from modelscope.utils.file_utils import get_default_modelscope_cache_dir +from modelscope.utils.file_utils import get_modelscope_cache_dir from modelscope.utils.logger import get_logger from modelscope.utils.thread_utils import thread_executor @@ -222,7 +222,7 @@ def _snapshot_download( temporary_cache_dir, cache = create_temporary_directory_and_cache( repo_id, local_dir=local_dir, cache_dir=cache_dir, repo_type=repo_type) - system_cache = cache_dir if cache_dir is not None else get_default_modelscope_cache_dir( + system_cache = cache_dir if cache_dir is not None else get_modelscope_cache_dir( ) if local_files_only: if len(cache.cached_files) == 0: From 40a57b6a7d3c27d4bae8a1038bbe7954eea2a249 Mon Sep 17 00:00:00 2001 From: tastelikefeet <58414341+tastelikefeet@users.noreply.github.com> Date: Thu, 6 Feb 2025 18:22:29 +0800 Subject: [PATCH 03/34] fix create_commit login (#1210) (cherry picked from commit 1cf7f4ff525e8f711d31462986069410fb6023a6) --- modelscope/hub/api.py | 87 +++++++++++++++++------------ modelscope/utils/hf_util/patcher.py | 26 ++------- tests/utils/test_hf_util.py | 1 + 3 files changed, 57 insertions(+), 57 deletions(-) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 02e02650..88875bfc 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -497,7 +497,7 @@ class HubApi: raise_for_http_status(r) return None - def _check_cookie(self, use_cookies: Union[bool, CookieJar] = False) -> CookieJar: # noqa + def _check_cookie(self, use_cookies: Union[bool, CookieJar] = False) -> CookieJar: # noqa cookies = None if isinstance(use_cookies, CookieJar): cookies = use_cookies @@ -1212,10 +1212,7 @@ class HubApi: if not repo_id: raise ValueError('Repo id cannot be empty!') - if token: - self.login(access_token=token) - else: - logger.warning('No token provided, will use the cached token.') + self.login(access_token=token) repo_id_list = repo_id.split('/') if len(repo_id_list) != 2: @@ -1287,8 +1284,7 @@ class HubApi: commit_message = commit_message or f'Commit to {repo_id}' commit_description = commit_description or '' - if token: - self.login(access_token=token) + self.login(access_token=token) # Construct payload payload = self._prepare_commit_payload( @@ -1361,8 +1357,7 @@ class HubApi: repo_type=repo_type, ) - if token: - self.login(access_token=token) + self.login(access_token=token) commit_message = ( commit_message if commit_message is not None else f'Upload {path_in_repo} to ModelScope hub' @@ -1414,7 +1409,7 @@ class HubApi: self, *, repo_id: str, - folder_path: Union[str, Path], + folder_path: Union[str, Path, List[str], List[Path]] = None, path_in_repo: Optional[str] = '', commit_message: Optional[str] = None, commit_description: Optional[str] = None, @@ -1423,16 +1418,14 @@ class HubApi: allow_patterns: Optional[Union[List[str], str]] = None, ignore_patterns: Optional[Union[List[str], str]] = None, max_workers: int = DEFAULT_MAX_WORKERS, + revision: Optional[str] = DEFAULT_REPOSITORY_REVISION, ) -> CommitInfo: - if repo_type not in REPO_TYPE_SUPPORT: raise ValueError(f'Invalid repo type: {repo_type}, supported repos: {REPO_TYPE_SUPPORT}') allow_patterns = allow_patterns if allow_patterns else None ignore_patterns = ignore_patterns if ignore_patterns else None - self.upload_checker.check_folder(folder_path) - # Ignore .git folder if ignore_patterns is None: ignore_patterns = [] @@ -1440,24 +1433,23 @@ class HubApi: ignore_patterns = [ignore_patterns] ignore_patterns += DEFAULT_IGNORE_PATTERNS - if token: - self.login(access_token=token) + self.login(access_token=token) commit_message = ( - commit_message if commit_message is not None else f'Upload folder to {repo_id} on ModelScope hub' + commit_message if commit_message is not None else f'Upload to {repo_id} on ModelScope hub' ) - commit_description = commit_description or 'Uploading folder' + commit_description = commit_description or 'Uploading files' # Get the list of files to upload, e.g. [('data/abc.png', '/path/to/abc.png'), ...] - prepared_repo_objects = HubApi._prepare_upload_folder( - folder_path=folder_path, + prepared_repo_objects = self._prepare_upload_folder( + folder_path_or_files=folder_path, path_in_repo=path_in_repo, allow_patterns=allow_patterns, ignore_patterns=ignore_patterns, ) self.upload_checker.check_normal_files( - file_path_list = [item for _, item in prepared_repo_objects], + file_path_list=[item for _, item in prepared_repo_objects], repo_type=repo_type, ) @@ -1526,6 +1518,7 @@ class HubApi: commit_description=commit_description, token=token, repo_type=repo_type, + revision=revision, ) return commit_info @@ -1668,7 +1661,7 @@ class HubApi: resp = response.json() raise_on_error(resp) - upload_objects = [] # list of objects to upload, [{'url': 'xxx', 'oid': 'xxx'}, ...] + upload_objects = [] # list of objects to upload, [{'url': 'xxx', 'oid': 'xxx'}, ...] resp_objects = resp['Data']['objects'] for obj in resp_objects: upload_objects.append( @@ -1678,24 +1671,44 @@ class HubApi: return upload_objects - @staticmethod def _prepare_upload_folder( - folder_path: Union[str, Path], - path_in_repo: str, - allow_patterns: Optional[Union[List[str], str]] = None, - ignore_patterns: Optional[Union[List[str], str]] = None, + self, + folder_path_or_files: Union[str, Path, List[str], List[Path]], + path_in_repo: str, + allow_patterns: Optional[Union[List[str], str]] = None, + ignore_patterns: Optional[Union[List[str], str]] = None, ) -> List[Union[tuple, list]]: + folder_path = None + files_path = None + if isinstance(folder_path_or_files, list): + if os.path.isfile(folder_path_or_files[0]): + files_path = folder_path_or_files + else: + raise ValueError('Uploading multiple folders is not supported now.') + else: + if os.path.isfile(folder_path_or_files): + files_path = [folder_path_or_files] + else: + folder_path = folder_path_or_files - folder_path = Path(folder_path).expanduser().resolve() - if not folder_path.is_dir(): - raise ValueError(f"Provided path: '{folder_path}' is not a directory") + if files_path is None: + self.upload_checker.check_folder(folder_path) + folder_path = Path(folder_path).expanduser().resolve() + if not folder_path.is_dir(): + raise ValueError(f"Provided path: '{folder_path}' is not a directory") - # List files from folder - relpath_to_abspath = { - path.relative_to(folder_path).as_posix(): path - for path in sorted(folder_path.glob('**/*')) # sorted to be deterministic - if path.is_file() - } + # List files from folder + relpath_to_abspath = { + path.relative_to(folder_path).as_posix(): path + for path in sorted(folder_path.glob('**/*')) # sorted to be deterministic + if path.is_file() + } + else: + relpath_to_abspath = {} + for path in files_path: + if os.path.isfile(path): + self.upload_checker.check_file(path) + relpath_to_abspath[os.path.basename(path)] = path # Filter files filtered_repo_objects = list( @@ -2004,5 +2017,5 @@ class UploadingCheck: total_size = sum([get_file_size(item) for item in normal_file_list]) if total_size > self.normal_file_size_total_limit: - raise ValueError(f'Total size of non-lfs files {total_size/(1024 * 1024)}MB ' - f'and exceeds limit: {self.normal_file_size_total_limit/(1024 * 1024)}MB') + raise ValueError(f'Total size of non-lfs files {total_size / (1024 * 1024)}MB ' + f'and exceeds limit: {self.normal_file_size_total_limit / (1024 * 1024)}MB') diff --git a/modelscope/utils/hf_util/patcher.py b/modelscope/utils/hf_util/patcher.py index 0529084c..43933ca9 100644 --- a/modelscope/utils/hf_util/patcher.py +++ b/modelscope/utils/hf_util/patcher.py @@ -466,30 +466,16 @@ def _patch_hub(): if any(['Add' not in op.__class__.__name__ for op in operations]): raise ValueError( 'ModelScope create_commit only support Add operation for now.') - ms_operations = [] - for op in operations: - _op = CommitOperationAdd( - path_in_repo=op.path_in_repo, - path_or_fileobj=op.path_or_fileobj) - _op._upload_mode = op._upload_mode - if any([ - re.search(pattern, _op.path_in_repo or _op.path_or_fileobj) - is not None for pattern in ignore_file_pattern - ]): - _op._upload_mode = 'lfs' - else: - _op._upload_mode = 'normal' - ms_operations.append(_op) - operations = ms_operations - return api.create_commit( - repo_id, - operations, + + all_files = [op.path_or_fileobj for op in operations] + api.upload_folder( + repo_id=repo_id, + folder_path=all_files, commit_message=commit_message, commit_description=commit_description, token=token, - repo_type=repo_type, revision=revision, - ) + repo_type=repo_type or 'model') # Patch repocard.validate from huggingface_hub import repocard diff --git a/tests/utils/test_hf_util.py b/tests/utils/test_hf_util.py index 84859f93..9826d991 100644 --- a/tests/utils/test_hf_util.py +++ b/tests/utils/test_hf_util.py @@ -227,6 +227,7 @@ class HFUtilTest(unittest.TestCase): from huggingface_hub import whoami self.assertTrue(whoami()['name'] == self.user) + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_push_to_hub(self): with patch_context(): from transformers import AutoModelForCausalLM From c220b4b5ddca2a846e2da8d736e5de03fec62a2a Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Fri, 7 Feb 2025 14:03:30 +0800 Subject: [PATCH 04/34] fix cache bust --- docker/Dockerfile.ubuntu | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu index 0ec13d12..186bb94b 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.ubuntu @@ -45,8 +45,9 @@ else \ pip cache purge; \ fi -RUN echo "cache bust $(date +%Y%m%d%H%M%S)" && \ - sh /tmp/install.sh {version_args} && \ +RUN echo "cache bust $(date +%Y%m%d%H%M%S)" + +RUN sh /tmp/install.sh {version_args} && \ curl -fsSL https://ollama.com/install.sh | sh && \ pip install --no-cache-dir -U funasr scikit-learn && \ pip install --no-cache-dir -U qwen_vl_utils pyav librosa timm transformers accelerate peft trl safetensors && \ From 9dda88a39978471f8e280b3385516443b2fb1564 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Fri, 7 Feb 2025 15:58:35 +0800 Subject: [PATCH 05/34] fix docker build --- docker/install.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/install.sh b/docker/install.sh index d7d367dc..ee747d20 100644 --- a/docker/install.sh +++ b/docker/install.sh @@ -8,12 +8,14 @@ lmdeploy_version=${5:-0.6.1} autogptq_version=${6:-0.7.1} flashattn_version=${7:-2.7.1.post4} -pip install --no-cache-dir -U autoawq lmdeploy==$lmdeploy_version - pip uninstall -y torch torchvision torchaudio pip install --no-cache-dir torch==$torch_version torchvision==$torchvision_version torchaudio==$torchaudio_version +pip install --no-cache-dir -U autoawq lmdeploy==$lmdeploy_version + +pip install --no-cache-dir torch==$torch_version torchvision==$torchvision_version torchaudio==$torchaudio_version + pip install --no-cache-dir tiktoken transformers_stream_generator bitsandbytes deepspeed torchmetrics decord optimum # pip install https://github.com/Dao-AILab/flash-attention/releases/download/v2.6.3/flash_attn-2.6.3+cu123torch2.4cxx11abiTRUE-cp310-cp310-linux_x86_64.whl From 2df80c0c40a6470812753dd080ea10f14e9e4aff Mon Sep 17 00:00:00 2001 From: Yingda Chen Date: Fri, 7 Feb 2025 16:02:37 +0800 Subject: [PATCH 06/34] support multiple include/exclude filter patterns in command line (#1214) Co-authored-by: Yingda Chen (cherry picked from commit 1f88654aa1b9808660075e06a6966b467f648f01) --- modelscope/cli/download.py | 9 +++++---- modelscope/cli/upload.py | 7 ++++--- modelscope/hub/utils/utils.py | 19 +++++++++++++++++++ tests/fileio/test_file.py | 16 ++++++++++++++++ 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/modelscope/cli/download.py b/modelscope/cli/download.py index 321c2b5d..6b430453 100644 --- a/modelscope/cli/download.py +++ b/modelscope/cli/download.py @@ -8,6 +8,7 @@ from modelscope.hub.file_download import (dataset_file_download, model_file_download) from modelscope.hub.snapshot_download import (dataset_snapshot_download, snapshot_download) +from modelscope.hub.utils.utils import convert_patterns from modelscope.utils.constant import DEFAULT_DATASET_REVISION @@ -141,8 +142,8 @@ class DownloadCMD(CLICommand): revision=self.args.revision, cache_dir=self.args.cache_dir, local_dir=self.args.local_dir, - allow_file_pattern=self.args.include, - ignore_file_pattern=self.args.exclude, + allow_file_pattern=convert_patterns(self.args.include), + ignore_file_pattern=convert_patterns(self.args.exclude), max_workers=self.args.max_workers, ) elif self.args.dataset: @@ -170,8 +171,8 @@ class DownloadCMD(CLICommand): revision=dataset_revision, cache_dir=self.args.cache_dir, local_dir=self.args.local_dir, - allow_file_pattern=self.args.include, - ignore_file_pattern=self.args.exclude, + allow_file_pattern=convert_patterns(self.args.include), + ignore_file_pattern=convert_patterns(self.args.exclude), max_workers=self.args.max_workers, ) else: diff --git a/modelscope/cli/upload.py b/modelscope/cli/upload.py index 29dacbe5..d32abdcc 100644 --- a/modelscope/cli/upload.py +++ b/modelscope/cli/upload.py @@ -4,6 +4,7 @@ from argparse import ArgumentParser, _SubParsersAction from modelscope.cli.base import CLICommand from modelscope.hub.api import HubApi, ModelScopeConfig +from modelscope.hub.utils.utils import convert_patterns, get_endpoint from modelscope.utils.constant import REPO_TYPE_MODEL, REPO_TYPE_SUPPORT @@ -89,7 +90,7 @@ class UploadCMD(CLICommand): parser.add_argument( '--endpoint', type=str, - default='https://www.modelscope.cn', + default=get_endpoint(), help='Endpoint for Modelscope service.') parser.set_defaults(func=subparser_func) @@ -166,8 +167,8 @@ class UploadCMD(CLICommand): commit_message=self.args.commit_message, commit_description=self.args.commit_description, repo_type=self.args.repo_type, - allow_patterns=self.args.include, - ignore_patterns=self.args.exclude, + allow_file_pattern=convert_patterns(self.args.include), + ignore_file_pattern=convert_patterns(self.args.exclude), max_workers=self.args.max_workers, ) else: diff --git a/modelscope/hub/utils/utils.py b/modelscope/hub/utils/utils.py index 3f3a4c75..3ad96fe2 100644 --- a/modelscope/hub/utils/utils.py +++ b/modelscope/hub/utils/utils.py @@ -31,6 +31,25 @@ def model_id_to_group_owner_name(model_id): return group_or_owner, name +def convert_patterns(raw_input: Union[str, List[str]]): + output = None + if isinstance(raw_input, str): + output = list() + if ',' in raw_input: + output = [s.strip() for s in raw_input.split(',')] + else: + output.append(raw_input.strip()) + elif isinstance(raw_input, list): + output = list() + for s in raw_input: + if isinstance(s, str): + if ',' in s: + output.extend([ss.strip() for ss in s.split(',')]) + else: + output.append(s.strip()) + return output + + # during model download, the '.' would be converted to '___' to produce # actual physical (masked) directory for storage def get_model_masked_directory(directory, model_id): diff --git a/tests/fileio/test_file.py b/tests/fileio/test_file.py index ded8ece7..383e8231 100644 --- a/tests/fileio/test_file.py +++ b/tests/fileio/test_file.py @@ -6,10 +6,26 @@ import unittest from requests import HTTPError from modelscope.fileio.file import File, HTTPStorage, LocalStorage +from modelscope.hub.utils.utils import convert_patterns class FileTest(unittest.TestCase): + def test_pattern_conversion(self): + self._assert_patterns(None, None) + self._assert_patterns('*.h5', ['*.h5']) + self._assert_patterns('*.h5 ', ['*.h5']) + self._assert_patterns('*.h5, *flax_model.msgpack', + ['*.h5', '*flax_model.msgpack']) + self._assert_patterns(['*.h5, *flax_model.msgpack'], + ['*.h5', '*flax_model.msgpack']) + self._assert_patterns(['*.h5 ', '*flax_model.msgpack'], + ['*.h5', '*flax_model.msgpack']) + + def _assert_patterns(self, raw_input, expected_output): + output = convert_patterns(raw_input) + self.assertEqual(expected_output, output) + def test_local_storage(self): storage = LocalStorage() temp_name = tempfile.gettempdir() + '/' + next( From 8282a236a054c2cf99350be2a49ef3826f6dbcb1 Mon Sep 17 00:00:00 2001 From: tastelikefeet <58414341+tastelikefeet@users.noreply.github.com> Date: Fri, 7 Feb 2025 17:31:32 +0800 Subject: [PATCH 07/34] Use legacy cache (#1215) (cherry picked from commit b5bb6d7bb0c964463fa1a2d7bce00846a1f1e107) --- docker/Dockerfile.ubuntu | 2 +- modelscope/hub/file_download.py | 41 ++++++++++++++++++++ modelscope/hub/snapshot_download.py | 3 -- modelscope/utils/hf_util/patcher.py | 59 +++++++++++++++++++---------- 4 files changed, 81 insertions(+), 24 deletions(-) diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu index 186bb94b..27e9af90 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.ubuntu @@ -67,5 +67,5 @@ RUN sh /tmp/install.sh {version_args} && \ ENV SETUPTOOLS_USE_DISTUTILS=stdlib ENV VLLM_USE_MODELSCOPE=True ENV LMDEPLOY_USE_MODELSCOPE=True -ENV MODELSCOPE_CACHE=/mnt/workspace/.cache/modelscope +ENV MODELSCOPE_CACHE=/mnt/workspace/.cache/modelscope/hub SHELL ["/bin/bash", "-c"] diff --git a/modelscope/hub/file_download.py b/modelscope/hub/file_download.py index 00eb8abf..ee0f5d89 100644 --- a/modelscope/hub/file_download.py +++ b/modelscope/hub/file_download.py @@ -4,6 +4,7 @@ import copy import hashlib import io import os +import shutil import tempfile import urllib import uuid @@ -286,6 +287,41 @@ def _repo_file_download( temporary_cache_dir, cache, headers, cookies) +def move_legacy_cache_to_standard_dir(cache_dir: str, model_id: str): + if cache_dir.endswith(os.path.sep): + cache_dir = cache_dir.strip(os.path.sep) + legacy_cache_root = os.path.dirname(cache_dir) + base_name = os.path.basename(cache_dir) + if base_name == 'datasets': + # datasets will not be not affected + return + if not legacy_cache_root.endswith('hub'): + # Two scenarios: + # We have restructured ModelScope cache directory, + # Scenery 1: + # When MODELSCOPE_CACHE is not set, the default directory remains + # the same at ~/.cache/modelscope/hub + # Scenery 2: + # When MODELSCOPE_CACHE is not set, the cache directory is moved from + # $MODELSCOPE_CACHE/hub to $MODELSCOPE_CACHE/. In this case, + # we will be migrating the hub directory accordingly. + legacy_cache_root = os.path.join(legacy_cache_root, 'hub') + group_or_owner, name = model_id_to_group_owner_name(model_id) + name = name.replace('.', '___') + temporary_cache_dir = os.path.join(cache_dir, group_or_owner, name) + legacy_cache_dir = os.path.join(legacy_cache_root, group_or_owner, name) + if os.path.exists( + legacy_cache_dir) and not os.path.exists(temporary_cache_dir): + logger.info( + f'Legacy cache dir exists: {legacy_cache_dir}, move to {temporary_cache_dir}' + ) + try: + shutil.move(legacy_cache_dir, temporary_cache_dir) + except Exception: # noqa + # Failed, skip + pass + + def create_temporary_directory_and_cache(model_id: str, local_dir: str = None, cache_dir: str = None, @@ -294,6 +330,10 @@ def create_temporary_directory_and_cache(model_id: str, default_cache_root = get_model_cache_root() elif repo_type == REPO_TYPE_DATASET: default_cache_root = get_dataset_cache_root() + else: + raise ValueError( + f'repo_type only support model and dataset, but now is : {repo_type}' + ) group_or_owner, name = model_id_to_group_owner_name(model_id) if local_dir is not None: @@ -302,6 +342,7 @@ def create_temporary_directory_and_cache(model_id: str, else: if cache_dir is None: cache_dir = default_cache_root + move_legacy_cache_to_standard_dir(cache_dir, model_id) if isinstance(cache_dir, Path): cache_dir = str(cache_dir) temporary_cache_dir = os.path.join(cache_dir, TEMPORARY_FOLDER_NAME, diff --git a/modelscope/hub/snapshot_download.py b/modelscope/hub/snapshot_download.py index 77b49847..2c79050c 100644 --- a/modelscope/hub/snapshot_download.py +++ b/modelscope/hub/snapshot_download.py @@ -17,7 +17,6 @@ from modelscope.hub.utils.utils import (get_model_masked_directory, model_id_to_group_owner_name) from modelscope.utils.constant import (DEFAULT_DATASET_REVISION, DEFAULT_MODEL_REVISION, - DEFAULT_REPOSITORY_REVISION, REPO_TYPE_DATASET, REPO_TYPE_MODEL, REPO_TYPE_SUPPORT) from modelscope.utils.file_utils import get_modelscope_cache_dir @@ -246,7 +245,6 @@ def _snapshot_download( _api = HubApi() if cookies is None: cookies = ModelScopeConfig.get_cookies() - repo_files = [] if repo_type == REPO_TYPE_MODEL: directory = os.path.abspath( local_dir) if local_dir is not None else os.path.join( @@ -313,7 +311,6 @@ def _snapshot_download( local_dir) if local_dir else os.path.join( system_cache, 'datasets', *repo_id.split('/')) print(f'Downloading Dataset to directory: {directory}') - group_or_owner, name = model_id_to_group_owner_name(repo_id) revision_detail = revision or DEFAULT_DATASET_REVISION diff --git a/modelscope/utils/hf_util/patcher.py b/modelscope/utils/hf_util/patcher.py index 43933ca9..74264c13 100644 --- a/modelscope/utils/hf_util/patcher.py +++ b/modelscope/utils/hf_util/patcher.py @@ -47,29 +47,48 @@ def get_all_imported_modules(): pass if importlib.util.find_spec('peft') is not None: - import peft - attributes = dir(peft) - imports = [attr for attr in attributes if not attr.startswith('__')] - all_imported_modules.extend( - [getattr(peft, _import) for _import in imports]) + try: + import peft + except: # noqa + pass + else: + attributes = dir(peft) + imports = [ + attr for attr in attributes if not attr.startswith('__') + ] + all_imported_modules.extend( + [getattr(peft, _import) for _import in imports]) if importlib.util.find_spec('diffusers') is not None: - import diffusers - if importlib.util.find_spec('diffusers') is not None: + try: + import diffusers + except: # noqa + pass + else: lazy_module = sys.modules['diffusers'] - _import_structure = lazy_module._import_structure - for key in _import_structure: - values = _import_structure[key] - for value in values: - if any([name in value - for name in diffusers_include_names]): - try: - module = importlib.import_module( - f'.{key}', diffusers.__name__) - value = getattr(module, value) - all_imported_modules.append(value) - except (ImportError, AttributeError): - pass + if hasattr(lazy_module, '_import_structure'): + _import_structure = lazy_module._import_structure + for key in _import_structure: + values = _import_structure[key] + for value in values: + if any([ + name in value + for name in diffusers_include_names + ]): + try: + module = importlib.import_module( + f'.{key}', diffusers.__name__) + value = getattr(module, value) + all_imported_modules.append(value) + except (ImportError, AttributeError): + pass + else: + attributes = dir(lazy_module) + imports = [ + attr for attr in attributes if not attr.startswith('__') + ] + all_imported_modules.extend( + [getattr(lazy_module, _import) for _import in imports]) return all_imported_modules From 27c57d0ca12e7778cfb28d171be3b629a963ceaf Mon Sep 17 00:00:00 2001 From: Yingda Chen Date: Fri, 7 Feb 2025 18:27:26 +0800 Subject: [PATCH 08/34] fix name (#1216) Co-authored-by: Yingda Chen (cherry picked from commit 555d002baee436c6ecb264020cb7d941bfa55bda) --- modelscope/cli/upload.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modelscope/cli/upload.py b/modelscope/cli/upload.py index d32abdcc..453a6314 100644 --- a/modelscope/cli/upload.py +++ b/modelscope/cli/upload.py @@ -167,8 +167,8 @@ class UploadCMD(CLICommand): commit_message=self.args.commit_message, commit_description=self.args.commit_description, repo_type=self.args.repo_type, - allow_file_pattern=convert_patterns(self.args.include), - ignore_file_pattern=convert_patterns(self.args.exclude), + allow_patterns=convert_patterns(self.args.include), + ignore_patterns=convert_patterns(self.args.exclude), max_workers=self.args.max_workers, ) else: From a0c91ba2f265cdd79690059d8327b7b49958b1da Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Sat, 8 Feb 2025 10:28:25 +0800 Subject: [PATCH 09/34] upgrade timm version --- docker/Dockerfile.ubuntu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu index 27e9af90..3eb90862 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.ubuntu @@ -59,7 +59,7 @@ RUN sh /tmp/install.sh {version_args} && \ pip install .[eval] && pip install evalscope -U --no-dependencies && pip install xtuner --no-dependencies && \ cd / && rm -fr /tmp/ms-swift && pip cache purge; \ pip install --no-cache-dir torch=={torch_version} torchvision=={torchvision_version} torchaudio=={torchaudio_version} {index_url} && \ - pip install --no-cache-dir transformers huggingface-hub==0.25.* -U && pip cache purge; \ + pip install --no-cache-dir transformers huggingface-hub==0.25.* -U && pip install --no-cache-dr timm>=0.9.0 && pip cache purge; \ pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \ pip config set install.trusted-host mirrors.aliyun.com && \ cp /tmp/resources/ubuntu2204.aliyun /etc/apt/sources.list From 388ecb72084a146805618ec8991edbb69cfa52e7 Mon Sep 17 00:00:00 2001 From: Yingda Chen Date: Sat, 8 Feb 2025 14:47:35 +0800 Subject: [PATCH 10/34] fix path name for log accuracy (#1217) * change log msg --------- Co-authored-by: Yingda Chen --- modelscope/hub/snapshot_download.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/modelscope/hub/snapshot_download.py b/modelscope/hub/snapshot_download.py index 2c79050c..75bcb991 100644 --- a/modelscope/hub/snapshot_download.py +++ b/modelscope/hub/snapshot_download.py @@ -246,9 +246,13 @@ def _snapshot_download( if cookies is None: cookies = ModelScopeConfig.get_cookies() if repo_type == REPO_TYPE_MODEL: - directory = os.path.abspath( - local_dir) if local_dir is not None else os.path.join( - system_cache, 'models', *repo_id.split('/')) + if local_dir: + directory = os.path.abspath(local_dir) + elif cache_dir: + directory = os.path.join(system_cache, *repo_id.split('/')) + else: + directory = os.path.join(system_cache, 'models', + *repo_id.split('/')) print(f'Downloading Model to directory: {directory}') revision_detail = _api.get_valid_revision_detail( repo_id, revision=revision, cookies=cookies) @@ -307,9 +311,13 @@ def _snapshot_download( ) elif repo_type == REPO_TYPE_DATASET: - directory = os.path.abspath( - local_dir) if local_dir else os.path.join( - system_cache, 'datasets', *repo_id.split('/')) + if local_dir: + directory = os.path.abspath(local_dir) + elif cache_dir: + directory = os.path.join(system_cache, *repo_id.split('/')) + else: + directory = os.path.join(system_cache, 'datasets', + *repo_id.split('/')) print(f'Downloading Dataset to directory: {directory}') group_or_owner, name = model_id_to_group_owner_name(repo_id) revision_detail = revision or DEFAULT_DATASET_REVISION From a94bbc2f1e0305ca0e671eadb12c031384e5e6d1 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Mon, 10 Feb 2025 10:50:36 +0800 Subject: [PATCH 11/34] update version (cherry picked from commit d501fd405b82ef7070250f0819fe12338e4cf903) --- docker/build_image.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/build_image.py b/docker/build_image.py index 7c8e0808..d614f521 100644 --- a/docker/build_image.py +++ b/docker/build_image.py @@ -265,15 +265,15 @@ class LLMImageBuilder(Builder): # A mirrored image of nvidia/cuda:12.4.0-devel-ubuntu22.04 args.base_image = 'nvidia/cuda:12.4.0-devel-ubuntu22.04' if not args.torch_version: - args.torch_version = '2.4.0' - args.torchaudio_version = '2.4.0' - args.torchvision_version = '0.19.0' + args.torch_version = '2.5.1' + args.torchaudio_version = '2.5.1' + args.torchvision_version = '0.20.1' if not args.cuda_version: args.cuda_version = '12.4.0' if not args.vllm_version: - args.vllm_version = '0.6.3.post1' + args.vllm_version = '0.7.2' if not args.lmdeploy_version: - args.lmdeploy_version = '0.6.2' + args.lmdeploy_version = '0.7.0.post2' if not args.autogptq_version: args.autogptq_version = '0.7.1' if not args.flashattn_version: From 7cc885af3f1a1747687ca01f2d68ff3f94db5f21 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Mon, 10 Feb 2025 18:39:29 +0800 Subject: [PATCH 12/34] fix import --- modelscope/utils/hf_util/patcher.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modelscope/utils/hf_util/patcher.py b/modelscope/utils/hf_util/patcher.py index 74264c13..6cb4ac4b 100644 --- a/modelscope/utils/hf_util/patcher.py +++ b/modelscope/utils/hf_util/patcher.py @@ -43,7 +43,7 @@ def get_all_imported_modules(): f'.{key}', transformers.__name__) value = getattr(module, value) all_imported_modules.append(value) - except (ImportError, AttributeError): + except: # noqa pass if importlib.util.find_spec('peft') is not None: @@ -80,7 +80,7 @@ def get_all_imported_modules(): f'.{key}', diffusers.__name__) value = getattr(module, value) all_imported_modules.append(value) - except (ImportError, AttributeError): + except: # noqa pass else: attributes = dir(lazy_module) @@ -250,7 +250,7 @@ def _patch_pretrained_class(all_imported_modules, wrap=False): has_from_pretrained = hasattr(var, 'from_pretrained') has_get_peft_type = hasattr(var, '_get_peft_type') has_get_config_dict = hasattr(var, 'get_config_dict') - except ImportError: + except: # noqa continue if wrap: @@ -260,7 +260,7 @@ def _patch_pretrained_class(all_imported_modules, wrap=False): else: all_available_modules.append( get_wrapped_class(var, **ignore_file_pattern_kwargs)) - except Exception: + except: # noqa all_available_modules.append(var) else: if has_from_pretrained and not hasattr(var, @@ -307,7 +307,7 @@ def _unpatch_pretrained_class(all_imported_modules): has_from_pretrained = hasattr(var, 'from_pretrained') has_get_peft_type = hasattr(var, '_get_peft_type') has_get_config_dict = hasattr(var, 'get_config_dict') - except ImportError: + except: # noqa continue if has_from_pretrained and hasattr(var, '_from_pretrained_origin'): var.from_pretrained = var._from_pretrained_origin From 6d1598fcfc45d75f57011a3f8e2c125cc60e556e Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Mon, 10 Feb 2025 18:52:55 +0800 Subject: [PATCH 13/34] enable cache --- docker/Dockerfile.ubuntu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu index 3eb90862..7d480388 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.ubuntu @@ -45,7 +45,7 @@ else \ pip cache purge; \ fi -RUN echo "cache bust $(date +%Y%m%d%H%M%S)" +RUN echo "cache bust $(date +%Y%m%d%H%M%S)" && true RUN sh /tmp/install.sh {version_args} && \ curl -fsSL https://ollama.com/install.sh | sh && \ From 4f555551a1dc3e06068a1d8cfa3de7c5d0083791 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Wed, 12 Feb 2025 14:55:12 +0800 Subject: [PATCH 14/34] remove requirements to avoid a low version of timm --- .../robust_image_classification/easyrobust_model.py | 12 ++++++++++-- requirements/cv.txt | 1 - 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/modelscope/models/cv/robust_image_classification/easyrobust_model.py b/modelscope/models/cv/robust_image_classification/easyrobust_model.py index 96c0d391..1feb9e86 100644 --- a/modelscope/models/cv/robust_image_classification/easyrobust_model.py +++ b/modelscope/models/cv/robust_image_classification/easyrobust_model.py @@ -4,11 +4,14 @@ import os import torch import torch.nn as nn +from modelscope import get_logger from modelscope.metainfo import Models from modelscope.models.base.base_torch_model import TorchModel from modelscope.models.builder import MODELS from modelscope.utils.constant import ModelFile, Tasks +logger = get_logger() + def normalize_fn(tensor, mean, std): """Differentiable version of torchvision.functional.normalize""" @@ -41,10 +44,15 @@ class NormalizeByChannelMeanStd(nn.Module): class EasyRobustModel(TorchModel): def __init__(self, model_dir: str, **kwargs): - import easyrobust.models + try: + import easyrobust.models + except ImportError as e: + logger.error( + 'You are using `EasyRobustModel`, but this model requires `easyrobust`,' + 'please install it with command `pip install easyrobust`') + raise e from timm.models import create_model from mmcls.datasets import ImageNet - import modelscope.models.cv.image_classification.backbones from modelscope.utils.hub import read_config super().__init__(model_dir) diff --git a/requirements/cv.txt b/requirements/cv.txt index 842cded2..efc0d5aa 100644 --- a/requirements/cv.txt +++ b/requirements/cv.txt @@ -8,7 +8,6 @@ control_ldm ddpm_guided_diffusion diffusers easydict -easyrobust edit_distance face_alignment>=1.3.5 fairscale>=0.4.1 From 0a030953966e5a29533a200a51747b69f226b883 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Wed, 12 Feb 2025 19:00:07 +0800 Subject: [PATCH 15/34] fix bugs of main branch name and import modules --- modelscope/hub/api.py | 2 +- modelscope/hub/push_to_hub.py | 5 ++- modelscope/utils/hf_util/patcher.py | 50 +++++++++++++++++++++++++---- 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 88875bfc..c49ac8a9 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -291,7 +291,7 @@ class HubApi: Returns: True if the repository exists, False otherwise. """ - if (repo_type is not None) and repo_type.lower != REPO_TYPE_MODEL: + if (repo_type is not None) and repo_type.lower() != REPO_TYPE_MODEL: raise Exception('Not support repo-type: %s' % repo_type) if (repo_id is None) or repo_id.count('/') != 1: raise Exception('Invalid repo_id: %s, must be of format namespace/name' % repo_type) diff --git a/modelscope/hub/push_to_hub.py b/modelscope/hub/push_to_hub.py index 3dc70b1d..df49ae5e 100644 --- a/modelscope/hub/push_to_hub.py +++ b/modelscope/hub/push_to_hub.py @@ -51,7 +51,10 @@ def _push_files_to_hub( with tempfile.TemporaryDirectory() as temp_cache_dir: from modelscope.hub.repository import Repository repo = Repository(temp_cache_dir, repo_id, revision=revision) - sub_folder = os.path.join(temp_cache_dir, path_in_repo) + if path_in_repo: + sub_folder = os.path.join(temp_cache_dir, path_in_repo) + else: + sub_folder = temp_cache_dir os.makedirs(sub_folder, exist_ok=True) if os.path.isfile(path_or_fileobj): dest_file = os.path.join(sub_folder, diff --git a/modelscope/utils/hf_util/patcher.py b/modelscope/utils/hf_util/patcher.py index 6cb4ac4b..e0b32e55 100644 --- a/modelscope/utils/hf_util/patcher.py +++ b/modelscope/utils/hf_util/patcher.py @@ -26,7 +26,8 @@ def get_all_imported_modules(): all_imported_modules = [] transformers_include_names = [ 'Auto', 'T5', 'BitsAndBytes', 'GenerationConfig', 'Quant', 'Awq', - 'GPTQ', 'BatchFeature', 'Qwen', 'Llama' + 'GPTQ', 'BatchFeature', 'Qwen', 'Llama', 'PretrainedConfig', + 'PreTrainedTokenizer', 'PreTrainedModel', 'PreTrainedTokenizerFast' ] diffusers_include_names = ['Pipeline'] if importlib.util.find_spec('transformers') is not None: @@ -109,6 +110,8 @@ def _patch_pretrained_class(all_imported_modules, wrap=False): from modelscope import snapshot_download if not os.path.exists(pretrained_model_name_or_path): revision = kwargs.pop('revision', None) + if revision is None or revision == 'main': + revision = 'master' model_dir = snapshot_download( pretrained_model_name_or_path, revision=revision, @@ -345,6 +348,8 @@ def _patch_hub(): from modelscope.hub.api import HubApi api = HubApi() api.login(token) + if revision is None or revision == 'main': + revision = 'master' return api.file_exists(repo_id, filename, revision=revision) def _file_download(repo_id: str, @@ -374,6 +379,8 @@ def _patch_hub(): from modelscope import HubApi api = HubApi() api.login(token) + if revision is None or revision == 'main': + revision = 'master' return file_download( repo_id, file_path=os.path.join(subfolder, filename) @@ -431,6 +438,8 @@ def _patch_hub(): **kwargs, ): from modelscope.hub.push_to_hub import _push_files_to_hub + if revision is None or revision == 'main': + revision = 'master' _push_files_to_hub( path_or_fileobj=folder_path, path_in_repo=path_in_repo, @@ -463,6 +472,8 @@ def _patch_hub(): commit_description: Optional[str] = None, **kwargs, ): + if revision is None or revision == 'main': + revision = 'master' from modelscope.hub.push_to_hub import _push_files_to_hub _push_files_to_hub(path_or_fileobj, path_in_repo, repo_id, token, revision, commit_message, commit_description) @@ -485,7 +496,8 @@ def _patch_hub(): if any(['Add' not in op.__class__.__name__ for op in operations]): raise ValueError( 'ModelScope create_commit only support Add operation for now.') - + if revision is None or revision == 'main': + revision = 'master' all_files = [op.path_or_fileobj for op in operations] api.upload_folder( repo_id=repo_id, @@ -500,14 +512,40 @@ def _patch_hub(): from huggingface_hub import repocard if not hasattr(repocard.RepoCard, '_validate_origin'): - def load(*args, **kwargs): - from huggingface_hub.errors import EntryNotFoundError - raise EntryNotFoundError(message='API not supported.') + def load( + cls, + repo_id_or_path: Union[str, Path], + repo_type: Optional[str] = None, + token: Optional[str] = None, + ignore_metadata_errors: bool = False, + ): + from modelscope.hub.api import HubApi + api = HubApi() + api.login(token) + if os.path.exists(repo_id_or_path): + file_path = repo_id_or_path + elif repo_type == 'model' or repo_type is None: + from modelscope import model_file_download + file_path = model_file_download(repo_id_or_path, 'README.md') + elif repo_type == 'dataset': + from modelscope import dataset_file_download + file_path = dataset_file_download(repo_id_or_path, 'README.md') + else: + raise ValueError( + f'repo_type should be `model` or `dataset`, but now is {repo_type}' + ) + + with open(file_path, 'r') as f: + repo_card = cls( + f.read(), ignore_metadata_errors=ignore_metadata_errors) + if not hasattr(repo_card.data, 'tags'): + repo_card.data.tags = [] + return repo_card repocard.RepoCard._validate_origin = repocard.RepoCard.validate repocard.RepoCard.validate = lambda *args, **kwargs: None repocard.RepoCard._load_origin = repocard.RepoCard.load - repocard.RepoCard.load = load + repocard.RepoCard.load = MethodType(load, repocard.RepoCard) if not hasattr(hf_api, '_hf_hub_download_origin'): # Patch hf_hub_download From cdb4dd34a7c9c06b2682922549d2034e3ee714d1 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Wed, 12 Feb 2025 19:31:40 +0800 Subject: [PATCH 16/34] support subfolder of from_pretrained --- modelscope/utils/hf_util/patcher.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modelscope/utils/hf_util/patcher.py b/modelscope/utils/hf_util/patcher.py index e0b32e55..be762f35 100644 --- a/modelscope/utils/hf_util/patcher.py +++ b/modelscope/utils/hf_util/patcher.py @@ -108,15 +108,23 @@ def _patch_pretrained_class(all_imported_modules, wrap=False): allow_file_pattern=None, **kwargs): from modelscope import snapshot_download + subfolder = kwargs.pop('subfolder', None) + file_filter = None + if subfolder: + file_filter = f'{subfolder}/*' if not os.path.exists(pretrained_model_name_or_path): revision = kwargs.pop('revision', None) if revision is None or revision == 'main': revision = 'master' + if file_filter is not None: + allow_file_pattern = file_filter model_dir = snapshot_download( pretrained_model_name_or_path, revision=revision, ignore_file_pattern=ignore_file_pattern, allow_file_pattern=allow_file_pattern) + if subfolder: + model_dir = os.path.join(model_dir, subfolder) else: model_dir = pretrained_model_name_or_path return model_dir From 005fdef5029ab13c8d6e2ee1f08c21cc92ed177f Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Wed, 12 Feb 2025 20:30:27 +0800 Subject: [PATCH 17/34] bust cache --- docker/Dockerfile.ubuntu | 3 ++- docker/build_image.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu index 7d480388..a294d2c0 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.ubuntu @@ -45,7 +45,8 @@ else \ pip cache purge; \ fi -RUN echo "cache bust $(date +%Y%m%d%H%M%S)" && true +ARG CUR_TIME={cur_time} +RUN echo $CUR_TIME RUN sh /tmp/install.sh {version_args} && \ curl -fsSL https://ollama.com/install.sh | sh && \ diff --git a/docker/build_image.py b/docker/build_image.py index d614f521..5f253eae 100644 --- a/docker/build_image.py +++ b/docker/build_image.py @@ -160,6 +160,7 @@ class CPUImageBuilder(Builder): content = content.replace('{extra_content}', extra_content) content = content.replace('{meta_file}', meta_file) content = content.replace('{version_args}', version_args) + content = content.replace('{cur_time}', formatted_time) content = content.replace('{install_ms_deps}', 'True') content = content.replace('{torch_version}', self.args.torch_version) @@ -222,6 +223,7 @@ RUN pip install tf-keras==2.16.0 --no-dependencies && \ content = content.replace('{extra_content}', extra_content) content = content.replace('{meta_file}', meta_file) content = content.replace('{version_args}', version_args) + content = content.replace('{cur_time}', formatted_time) content = content.replace('{install_ms_deps}', 'True') content = content.replace('{torch_version}', self.args.torch_version) @@ -296,6 +298,7 @@ class LLMImageBuilder(Builder): content = content.replace('{extra_content}', extra_content) content = content.replace('{meta_file}', meta_file) content = content.replace('{version_args}', version_args) + content = content.replace('{cur_time}', formatted_time) content = content.replace('{install_ms_deps}', 'False') content = content.replace('{torch_version}', self.args.torch_version) From 66ab912bc534d11e53e8c14296f6a32d8146069d Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Thu, 13 Feb 2025 14:02:53 +0800 Subject: [PATCH 18/34] fix parent-child classmethod problem --- modelscope/utils/hf_util/patcher.py | 53 ++++++++++++++++------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/modelscope/utils/hf_util/patcher.py b/modelscope/utils/hf_util/patcher.py index be762f35..5c0bb56e 100644 --- a/modelscope/utils/hf_util/patcher.py +++ b/modelscope/utils/hf_util/patcher.py @@ -129,30 +129,42 @@ def _patch_pretrained_class(all_imported_modules, wrap=False): model_dir = pretrained_model_name_or_path return model_dir - def patch_pretrained_model_name_or_path(pretrained_model_name_or_path, + def patch_pretrained_model_name_or_path(cls, pretrained_model_name_or_path, *model_args, **kwargs): - """Patch all from_pretrained/get_config_dict""" + """Patch all from_pretrained""" model_dir = get_model_dir(pretrained_model_name_or_path, kwargs.pop('ignore_file_pattern', None), kwargs.pop('allow_file_pattern', None), **kwargs) - return kwargs.pop('ori_func')(model_dir, *model_args, **kwargs) + return cls._from_pretrained_origin.__func__(cls, model_dir, + *model_args, **kwargs) - def patch_peft_model_id(model, model_id, *model_args, **kwargs): + def patch_get_config_dict(cls, pretrained_model_name_or_path, *model_args, + **kwargs): + """Patch all get_config_dict""" + model_dir = get_model_dir(pretrained_model_name_or_path, + kwargs.pop('ignore_file_pattern', None), + kwargs.pop('allow_file_pattern', None), + **kwargs) + return cls._get_config_dict_origin.__func__(cls, model_dir, + *model_args, **kwargs) + + def patch_peft_model_id(cls, model, model_id, *model_args, **kwargs): """Patch all peft.from_pretrained""" model_dir = get_model_dir(model_id, kwargs.pop('ignore_file_pattern', None), kwargs.pop('allow_file_pattern', None), **kwargs) - return kwargs.pop('ori_func')(model, model_dir, *model_args, **kwargs) + return cls._from_pretrained_origin.__func__(cls, model, model_dir, + *model_args, **kwargs) - def _get_peft_type(model_id, **kwargs): + def patch_get_peft_type(cls, model_id, **kwargs): """Patch all _get_peft_type""" model_dir = get_model_dir(model_id, kwargs.pop('ignore_file_pattern', None), kwargs.pop('allow_file_pattern', None), **kwargs) - return kwargs.pop('ori_func')(model_dir, **kwargs) + return cls._get_peft_type_origin.__func__(cls, model_dir, **kwargs) def get_wrapped_class( module_class: 'PreTrainedModel', @@ -281,29 +293,24 @@ def _patch_pretrained_class(all_imported_modules, wrap=False): is_peft = 'model' in parameters and 'model_id' in parameters var._from_pretrained_origin = var.from_pretrained if not is_peft: - var.from_pretrained = partial( - patch_pretrained_model_name_or_path, - ori_func=var._from_pretrained_origin, - **ignore_file_pattern_kwargs) + var.from_pretrained = classmethod( + partial(patch_pretrained_model_name_or_path, + **ignore_file_pattern_kwargs)) else: - var.from_pretrained = partial( - patch_peft_model_id, - ori_func=var._from_pretrained_origin, - **ignore_file_pattern_kwargs) + var.from_pretrained = classmethod( + partial(patch_peft_model_id, + **ignore_file_pattern_kwargs)) if has_get_peft_type and not hasattr(var, '_get_peft_type_origin'): var._get_peft_type_origin = var._get_peft_type - var._get_peft_type = partial( - _get_peft_type, - ori_func=var._get_peft_type_origin, - **ignore_file_pattern_kwargs) + var._get_peft_type = classmethod( + partial(patch_get_peft_type, **ignore_file_pattern_kwargs)) if has_get_config_dict and not hasattr(var, '_get_config_dict_origin'): var._get_config_dict_origin = var.get_config_dict - var.get_config_dict = partial( - patch_pretrained_model_name_or_path, - ori_func=var._get_config_dict_origin, - **ignore_file_pattern_kwargs) + var.get_config_dict = classmethod( + partial(patch_get_config_dict, + **ignore_file_pattern_kwargs)) all_available_modules.append(var) return all_available_modules From 73f58d9a8de2a399cce68c8d10444a485a35c4eb Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Thu, 13 Feb 2025 17:12:42 +0800 Subject: [PATCH 19/34] avoid massive imports --- modelscope/utils/hf_util/patcher.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/modelscope/utils/hf_util/patcher.py b/modelscope/utils/hf_util/patcher.py index 5c0bb56e..818acfc4 100644 --- a/modelscope/utils/hf_util/patcher.py +++ b/modelscope/utils/hf_util/patcher.py @@ -25,11 +25,11 @@ def get_all_imported_modules(): """Find all modules in transformers/peft/diffusers""" all_imported_modules = [] transformers_include_names = [ - 'Auto', 'T5', 'BitsAndBytes', 'GenerationConfig', 'Quant', 'Awq', - 'GPTQ', 'BatchFeature', 'Qwen', 'Llama', 'PretrainedConfig', + 'Auto.*', 'T5.*', 'BitsAndBytesConfig', 'GenerationConfig', 'Awq.*', + 'GPTQ.*', 'BatchFeature', 'Qwen.*', 'Llama.*', 'PretrainedConfig', 'PreTrainedTokenizer', 'PreTrainedModel', 'PreTrainedTokenizerFast' ] - diffusers_include_names = ['Pipeline'] + diffusers_include_names = ['^(?!TF|Flax).*Pipeline$'] if importlib.util.find_spec('transformers') is not None: import transformers lazy_module = sys.modules['transformers'] @@ -38,7 +38,10 @@ def get_all_imported_modules(): values = _import_structure[key] for value in values: # pretrained - if any([name in value for name in transformers_include_names]): + if any([ + re.fullmatch(name, value) + for name in transformers_include_names + ]): try: module = importlib.import_module( f'.{key}', transformers.__name__) @@ -73,7 +76,7 @@ def get_all_imported_modules(): values = _import_structure[key] for value in values: if any([ - name in value + re.fullmatch(name, value) for name in diffusers_include_names ]): try: From b2f8abe952fadf72ba72c9c597b9b98a2d1d3e69 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Thu, 13 Feb 2025 17:38:21 +0800 Subject: [PATCH 20/34] avoid throwing error when create repo --- modelscope/hub/api.py | 65 +++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index c49ac8a9..95007b1c 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -1225,29 +1225,31 @@ class HubApi: if visibility is None: raise ValueError(f'Invalid visibility: {visibility}, ' f'supported visibilities: `public`, `private`, `internal`') - repo_url: str = self.create_model( - model_id=repo_id, - visibility=visibility, - license=license, - chinese_name=chinese_name, - ) - - with tempfile.TemporaryDirectory() as temp_cache_dir: - from modelscope.hub.repository import Repository - repo = Repository(temp_cache_dir, repo_id) - default_config = { - 'framework': 'pytorch', - 'task': 'text-generation', - 'allow_remote': True - } - config_json = kwargs.get('config_json') - if not config_json: - config_json = {} - config = {**default_config, **config_json} - add_content_to_file( - repo, - 'configuration.json', [json.dumps(config)], - ignore_push_error=True) + if not self.repo_exists(repo_id, repo_type=repo_type): + repo_url: str = self.create_model( + model_id=repo_id, + visibility=visibility, + license=license, + chinese_name=chinese_name, + ) + with tempfile.TemporaryDirectory() as temp_cache_dir: + from modelscope.hub.repository import Repository + repo = Repository(temp_cache_dir, repo_id) + default_config = { + 'framework': 'pytorch', + 'task': 'text-generation', + 'allow_remote': True + } + config_json = kwargs.get('config_json') + if not config_json: + config_json = {} + config = {**default_config, **config_json} + add_content_to_file( + repo, + 'configuration.json', [json.dumps(config)], + ignore_push_error=True) + else: + repo_url = f'{self.endpoint}/{repo_id}' elif repo_type == REPO_TYPE_DATASET: visibilities = {k: v for k, v in DatasetVisibility.__dict__.items() if not k.startswith('__')} @@ -1255,13 +1257,16 @@ class HubApi: if visibility is None: raise ValueError(f'Invalid visibility: {visibility}, ' f'supported visibilities: `public`, `private`, `internal`') - repo_url: str = self.create_dataset( - dataset_name=repo_name, - namespace=namespace, - chinese_name=chinese_name, - license=license, - visibility=visibility, - ) + if not self.repo_exists(repo_id, repo_type=repo_type): + repo_url: str = self.create_dataset( + dataset_name=repo_name, + namespace=namespace, + chinese_name=chinese_name, + license=license, + visibility=visibility, + ) + else: + repo_url = f'{self.endpoint}/datasets/{namespace}/{repo_name}' else: raise ValueError(f'Invalid repo type: {repo_type}, supported repos: {REPO_TYPE_SUPPORT}') From e6db19c11f65943ae23a2454d58083502d7ef1ae Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Thu, 13 Feb 2025 20:52:08 +0800 Subject: [PATCH 21/34] safe import --- modelscope/utils/hf_util/auto_class.py | 14 +++-- modelscope/utils/hf_util/patcher.py | 82 +++++++++++++++----------- 2 files changed, 56 insertions(+), 40 deletions(-) diff --git a/modelscope/utils/hf_util/auto_class.py b/modelscope/utils/hf_util/auto_class.py index b07168bf..f2b2210e 100644 --- a/modelscope/utils/hf_util/auto_class.py +++ b/modelscope/utils/hf_util/auto_class.py @@ -75,8 +75,12 @@ if TYPE_CHECKING: else: from .patcher import get_all_imported_modules, _patch_pretrained_class - all_available_modules = _patch_pretrained_class( - get_all_imported_modules(), wrap=True) - - for module in all_available_modules: - globals()[module.__name__] = module + try: + all_available_modules = _patch_pretrained_class( + get_all_imported_modules(), wrap=True) + except Exception: # noqa + import traceback + traceback.print_exc() + else: + for module in all_available_modules: + globals()[module.__name__] = module diff --git a/modelscope/utils/hf_util/patcher.py b/modelscope/utils/hf_util/patcher.py index 818acfc4..28f8eeb5 100644 --- a/modelscope/utils/hf_util/patcher.py +++ b/modelscope/utils/hf_util/patcher.py @@ -29,12 +29,15 @@ def get_all_imported_modules(): 'GPTQ.*', 'BatchFeature', 'Qwen.*', 'Llama.*', 'PretrainedConfig', 'PreTrainedTokenizer', 'PreTrainedModel', 'PreTrainedTokenizerFast' ] + peft_include_names = ['.*PeftModel.*', '.*Config'] diffusers_include_names = ['^(?!TF|Flax).*Pipeline$'] if importlib.util.find_spec('transformers') is not None: import transformers lazy_module = sys.modules['transformers'] _import_structure = lazy_module._import_structure for key in _import_structure: + if 'dummy' in key.lower(): + continue values = _import_structure[key] for value in values: # pretrained @@ -60,8 +63,11 @@ def get_all_imported_modules(): imports = [ attr for attr in attributes if not attr.startswith('__') ] - all_imported_modules.extend( - [getattr(peft, _import) for _import in imports]) + all_imported_modules.extend([ + getattr(peft, _import) for _import in imports if any([ + re.fullmatch(name, _import) for name in peft_include_names + ]) + ]) if importlib.util.find_spec('diffusers') is not None: try: @@ -73,6 +79,8 @@ def get_all_imported_modules(): if hasattr(lazy_module, '_import_structure'): _import_structure = lazy_module._import_structure for key in _import_structure: + if 'dummy' in key.lower(): + continue values = _import_structure[key] for value in values: if any([ @@ -91,8 +99,13 @@ def get_all_imported_modules(): imports = [ attr for attr in attributes if not attr.startswith('__') ] - all_imported_modules.extend( - [getattr(lazy_module, _import) for _import in imports]) + all_imported_modules.extend([ + getattr(lazy_module, _import) for _import in imports + if any([ + re.fullmatch(name, _import) + for name in diffusers_include_names + ]) + ]) return all_imported_modules @@ -526,40 +539,39 @@ def _patch_hub(): revision=revision, repo_type=repo_type or 'model') + def load( + cls, + repo_id_or_path: Union[str, Path], + repo_type: Optional[str] = None, + token: Optional[str] = None, + ignore_metadata_errors: bool = False, + ): + from modelscope.hub.api import HubApi + api = HubApi() + api.login(token) + if os.path.exists(repo_id_or_path): + file_path = repo_id_or_path + elif repo_type == 'model' or repo_type is None: + from modelscope import model_file_download + file_path = model_file_download(repo_id_or_path, 'README.md') + elif repo_type == 'dataset': + from modelscope import dataset_file_download + file_path = dataset_file_download(repo_id_or_path, 'README.md') + else: + raise ValueError( + f'repo_type should be `model` or `dataset`, but now is {repo_type}' + ) + + with open(file_path, 'r') as f: + repo_card = cls( + f.read(), ignore_metadata_errors=ignore_metadata_errors) + if not hasattr(repo_card.data, 'tags'): + repo_card.data.tags = [] + return repo_card + # Patch repocard.validate from huggingface_hub import repocard if not hasattr(repocard.RepoCard, '_validate_origin'): - - def load( - cls, - repo_id_or_path: Union[str, Path], - repo_type: Optional[str] = None, - token: Optional[str] = None, - ignore_metadata_errors: bool = False, - ): - from modelscope.hub.api import HubApi - api = HubApi() - api.login(token) - if os.path.exists(repo_id_or_path): - file_path = repo_id_or_path - elif repo_type == 'model' or repo_type is None: - from modelscope import model_file_download - file_path = model_file_download(repo_id_or_path, 'README.md') - elif repo_type == 'dataset': - from modelscope import dataset_file_download - file_path = dataset_file_download(repo_id_or_path, 'README.md') - else: - raise ValueError( - f'repo_type should be `model` or `dataset`, but now is {repo_type}' - ) - - with open(file_path, 'r') as f: - repo_card = cls( - f.read(), ignore_metadata_errors=ignore_metadata_errors) - if not hasattr(repo_card.data, 'tags'): - repo_card.data.tags = [] - return repo_card - repocard.RepoCard._validate_origin = repocard.RepoCard.validate repocard.RepoCard.validate = lambda *args, **kwargs: None repocard.RepoCard._load_origin = repocard.RepoCard.load From 88e6de000d4ea9ba426cc1d3edc8220802223454 Mon Sep 17 00:00:00 2001 From: tastelikefeet <58414341+tastelikefeet@users.noreply.github.com> Date: Wed, 19 Feb 2025 18:11:23 +0800 Subject: [PATCH 22/34] Update omegaconf (#1234) (cherry picked from commit e733458746385e164458fb2ca6b0bcd404ece547) --- docker/Dockerfile.ubuntu | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu index a294d2c0..4f6186f0 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.ubuntu @@ -65,6 +65,8 @@ RUN sh /tmp/install.sh {version_args} && \ pip config set install.trusted-host mirrors.aliyun.com && \ cp /tmp/resources/ubuntu2204.aliyun /etc/apt/sources.list +RUN pip install --no-cache-dir omegaconf==2.3.0 && pip cache purge + ENV SETUPTOOLS_USE_DISTUTILS=stdlib ENV VLLM_USE_MODELSCOPE=True ENV LMDEPLOY_USE_MODELSCOPE=True From 5f00dfb8f91d1912a76facbad94aca904d2818bc Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Wed, 19 Feb 2025 18:13:13 +0800 Subject: [PATCH 23/34] bump version --- modelscope/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modelscope/version.py b/modelscope/version.py index 8984216b..fa49294a 100644 --- a/modelscope/version.py +++ b/modelscope/version.py @@ -1,5 +1,5 @@ # Make sure to modify __release_datetime__ to release time when making official release. -__version__ = '1.23.0' +__version__ = '1.23.1' # default release datetime for branches under active development is set # to be a time far-far-away-into-the-future -__release_datetime__ = '2025-02-10 00:00:00' +__release_datetime__ = '2025-02-20 24:00:00' From 8a93110bc2ef9fa229b7d0ff7e7f8609c9d490b3 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Thu, 20 Feb 2025 11:40:47 +0800 Subject: [PATCH 24/34] fix datetime --- modelscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modelscope/version.py b/modelscope/version.py index fa49294a..b7e4b2b3 100644 --- a/modelscope/version.py +++ b/modelscope/version.py @@ -2,4 +2,4 @@ __version__ = '1.23.1' # default release datetime for branches under active development is set # to be a time far-far-away-into-the-future -__release_datetime__ = '2025-02-20 24:00:00' +__release_datetime__ = '2025-02-21 00:00:00' From a1455d3b1142b46966e966e04a727c73cb0a4e2a Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Mon, 3 Mar 2025 17:09:46 +0800 Subject: [PATCH 25/34] test --- modelscope/hub/api.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 95007b1c..e2a5c137 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -1297,6 +1297,8 @@ class HubApi: commit_message=commit_message, ) + print(f'Got payload: {payload}') + # POST cookies = ModelScopeConfig.get_cookies() if cookies is None: @@ -1310,6 +1312,8 @@ class HubApi: resp = response.json() + print(f'Got resp: {resp}') + if not resp['Success']: commit_message = resp['Message'] logger.warning(f'{commit_message}') From 80ccd58d6154fad283a42492c473e4b97b72166d Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Mon, 3 Mar 2025 17:30:42 +0800 Subject: [PATCH 26/34] update --- modelscope/hub/api.py | 4 +++- modelscope/utils/repo_utils.py | 18 +++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index e2a5c137..8d9b35af 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -1289,6 +1289,8 @@ class HubApi: commit_message = commit_message or f'Commit to {repo_id}' commit_description = commit_description or '' + print(f'>>url: {url}, >>commit_message: {commit_message}, >>commit_description: {commit_description}') + self.login(access_token=token) # Construct payload @@ -1297,7 +1299,7 @@ class HubApi: commit_message=commit_message, ) - print(f'Got payload: {payload}') + print(f'>> Got payload: {payload}') # POST cookies = ModelScopeConfig.get_cookies() diff --git a/modelscope/utils/repo_utils.py b/modelscope/utils/repo_utils.py index 85ddc2f7..2958bc1c 100644 --- a/modelscope/utils/repo_utils.py +++ b/modelscope/utils/repo_utils.py @@ -244,14 +244,14 @@ class CommitInfo(str): Url to the PR that has been created, if any. Populated when `create_pr=True` is passed. - pr_revision (`str`, *optional*): - Revision of the PR that has been created, if any. Populated when - `create_pr=True` is passed. Example: `"refs/pr/1"`. + # pr_revision (`str`, *optional*): + # Revision of the PR that has been created, if any. Populated when + # `create_pr=True` is passed. Example: `"refs/pr/1"`. - pr_num (`int`, *optional*): - Number of the PR discussion that has been created, if any. Populated when - `create_pr=True` is passed. Can be passed as `discussion_num` in - [`get_discussion_details`]. Example: `1`. + # pr_num (`int`, *optional*): + # Number of the PR discussion that has been created, if any. Populated when + # `create_pr=True` is passed. Can be passed as `discussion_num` in + # [`get_discussion_details`]. Example: `1`. _url (`str`, *optional*): Legacy url for `str` compatibility. Can be the url to the uploaded file on the Hub (if returned by @@ -267,8 +267,8 @@ class CommitInfo(str): pr_url: Optional[str] = None # Computed from `pr_url` in `__post_init__` - pr_revision: Optional[str] = field(init=False) - pr_num: Optional[str] = field(init=False) + # pr_revision: Optional[str] = field(init=False) + # pr_num: Optional[str] = field(init=False) # legacy url for `str` compatibility (ex: url to uploaded file, url to uploaded folder, url to PR, etc.) _url: str = field( From 5d6cb351f182724f5a474801e5589386c797a9ea Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Mon, 3 Mar 2025 18:21:12 +0800 Subject: [PATCH 27/34] refine commit info --- modelscope/hub/api.py | 6 ------ modelscope/utils/repo_utils.py | 38 ++-------------------------------- 2 files changed, 2 insertions(+), 42 deletions(-) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 8d9b35af..95007b1c 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -1289,8 +1289,6 @@ class HubApi: commit_message = commit_message or f'Commit to {repo_id}' commit_description = commit_description or '' - print(f'>>url: {url}, >>commit_message: {commit_message}, >>commit_description: {commit_description}') - self.login(access_token=token) # Construct payload @@ -1299,8 +1297,6 @@ class HubApi: commit_message=commit_message, ) - print(f'>> Got payload: {payload}') - # POST cookies = ModelScopeConfig.get_cookies() if cookies is None: @@ -1314,8 +1310,6 @@ class HubApi: resp = response.json() - print(f'Got resp: {resp}') - if not resp['Success']: commit_message = resp['Message'] logger.warning(f'{commit_message}') diff --git a/modelscope/utils/repo_utils.py b/modelscope/utils/repo_utils.py index 2958bc1c..278217e3 100644 --- a/modelscope/utils/repo_utils.py +++ b/modelscope/utils/repo_utils.py @@ -220,7 +220,7 @@ class RepoUtils: @dataclass -class CommitInfo(str): +class CommitInfo: """Data structure containing information about a newly created commit. Returned by any method that creates a commit on the Hub: [`create_commit`], [`upload_file`], [`upload_folder`], @@ -240,46 +240,12 @@ class CommitInfo(str): oid (`str`): Commit hash id. Example: `"91c54ad1727ee830252e457677f467be0bfd8a57"`. - pr_url (`str`, *optional*): - Url to the PR that has been created, if any. Populated when `create_pr=True` - is passed. - - # pr_revision (`str`, *optional*): - # Revision of the PR that has been created, if any. Populated when - # `create_pr=True` is passed. Example: `"refs/pr/1"`. - - # pr_num (`int`, *optional*): - # Number of the PR discussion that has been created, if any. Populated when - # `create_pr=True` is passed. Can be passed as `discussion_num` in - # [`get_discussion_details`]. Example: `1`. - - _url (`str`, *optional*): - Legacy url for `str` compatibility. Can be the url to the uploaded file on the Hub (if returned by - [`upload_file`]), to the uploaded folder on the Hub (if returned by [`upload_folder`]) or to the commit on - the Hub (if returned by [`create_commit`]). Defaults to `commit_url`. It is deprecated to use this - attribute. Please use `commit_url` instead. """ commit_url: str commit_message: str commit_description: str oid: str - pr_url: Optional[str] = None - - # Computed from `pr_url` in `__post_init__` - # pr_revision: Optional[str] = field(init=False) - # pr_num: Optional[str] = field(init=False) - - # legacy url for `str` compatibility (ex: url to uploaded file, url to uploaded folder, url to PR, etc.) - _url: str = field( - repr=False, default=None) # type: ignore # defaults to `commit_url` - - def __new__(cls, - *args, - commit_url: str, - _url: Optional[str] = None, - **kwargs): - return str.__new__(cls, _url or commit_url) def to_dict(cls): return { @@ -287,7 +253,7 @@ class CommitInfo(str): 'commit_message': cls.commit_message, 'commit_description': cls.commit_description, 'oid': cls.oid, - 'pr_url': cls.pr_url, + # 'pr_url': cls.pr_url, } From 5737286386db70863bb49c74ec91c05d9ec4ae2f Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Mon, 3 Mar 2025 18:24:13 +0800 Subject: [PATCH 28/34] refine commit info --- modelscope/utils/repo_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modelscope/utils/repo_utils.py b/modelscope/utils/repo_utils.py index 278217e3..55d16251 100644 --- a/modelscope/utils/repo_utils.py +++ b/modelscope/utils/repo_utils.py @@ -253,7 +253,6 @@ class CommitInfo: 'commit_message': cls.commit_message, 'commit_description': cls.commit_description, 'oid': cls.oid, - # 'pr_url': cls.pr_url, } From 6c39f998e16381e81990120911ea9f58cf0fa2b4 Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Tue, 4 Mar 2025 20:02:30 +0800 Subject: [PATCH 29/34] update version and cherry-pick refine commit info --- modelscope/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modelscope/version.py b/modelscope/version.py index b7e4b2b3..a09bf7ea 100644 --- a/modelscope/version.py +++ b/modelscope/version.py @@ -1,5 +1,5 @@ # Make sure to modify __release_datetime__ to release time when making official release. -__version__ = '1.23.1' +__version__ = '1.23.2' # default release datetime for branches under active development is set # to be a time far-far-away-into-the-future -__release_datetime__ = '2025-02-21 00:00:00' +__release_datetime__ = '2025-03-04 21:00:00' From 30d8995cd9a567040dfc48e59e2697c25f523669 Mon Sep 17 00:00:00 2001 From: suluyan Date: Mon, 10 Mar 2025 15:56:52 +0800 Subject: [PATCH 30/34] rm import --- modelscope/utils/hf_util/pipeline_builder.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modelscope/utils/hf_util/pipeline_builder.py b/modelscope/utils/hf_util/pipeline_builder.py index 5386bead..4a37cd04 100644 --- a/modelscope/utils/hf_util/pipeline_builder.py +++ b/modelscope/utils/hf_util/pipeline_builder.py @@ -1,13 +1,12 @@ import os from typing import Optional, Union -import torch from transformers import Pipeline as PipelineHF from transformers import PreTrainedModel, TFPreTrainedModel, pipeline from transformers.pipelines import check_task, get_task from modelscope.hub import snapshot_download -from modelscope.utils.hf_util.patcher import _patch_pretrained_class, patch_hub +from modelscope.utils.hf_util.patcher import _patch_pretrained_class def _get_hf_device(device): From c45bc6d19f9ea988b482005ec7156b850665fa9b Mon Sep 17 00:00:00 2001 From: suluyan Date: Mon, 10 Mar 2025 16:29:02 +0800 Subject: [PATCH 31/34] rm import --- modelscope/utils/hf_util/pipeline_builder.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/modelscope/utils/hf_util/pipeline_builder.py b/modelscope/utils/hf_util/pipeline_builder.py index 4a37cd04..20ae0535 100644 --- a/modelscope/utils/hf_util/pipeline_builder.py +++ b/modelscope/utils/hf_util/pipeline_builder.py @@ -1,10 +1,6 @@ import os from typing import Optional, Union -from transformers import Pipeline as PipelineHF -from transformers import PreTrainedModel, TFPreTrainedModel, pipeline -from transformers.pipelines import check_task, get_task - from modelscope.hub import snapshot_download from modelscope.utils.hf_util.patcher import _patch_pretrained_class @@ -20,6 +16,7 @@ def _get_hf_device(device): def _get_hf_pipeline_class(task, model): + from transformers.pipelines import check_task, get_task if not task: task = get_task(model) normalized_task, targeted_task, task_options = check_task(task) @@ -34,7 +31,9 @@ def hf_pipeline( framework: Optional[str] = None, device: Optional[Union[int, str, 'torch.device']] = None, **kwargs, -) -> PipelineHF: +) -> 'transformers.Pipeline': + from transformers import pipeline + if isinstance(model, str): if not os.path.exists(model): model = snapshot_download(model) From 49b94409c1790cdf75f50ca2b51ca19241be7969 Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Tue, 11 Mar 2025 10:18:40 +0800 Subject: [PATCH 32/34] update --- modelscope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modelscope/version.py b/modelscope/version.py index defa20e8..031a86b4 100644 --- a/modelscope/version.py +++ b/modelscope/version.py @@ -2,4 +2,4 @@ __version__ = '2.0.0' # default release datetime for branches under active development is set # to be a time far-far-away-into-the-future -__release_datetime__ = '2099-09-06 00:00:00' \ No newline at end of file +__release_datetime__ = '2099-09-06 00:00:00' From 26ef8bb2295b876265ff1c1445a1e8b8f51a778e Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Tue, 11 Mar 2025 10:38:16 +0800 Subject: [PATCH 33/34] fix lint --- modelscope/utils/hf_util/patcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modelscope/utils/hf_util/patcher.py b/modelscope/utils/hf_util/patcher.py index 983faefe..b31b6e1d 100644 --- a/modelscope/utils/hf_util/patcher.py +++ b/modelscope/utils/hf_util/patcher.py @@ -620,7 +620,7 @@ def _patch_hub(): from huggingface_hub import repocard if not hasattr(repocard.RepoCard, '_validate_origin'): - def load(*args, **kwargs): + def load(*args, **kwargs): # noqa from huggingface_hub.errors import EntryNotFoundError raise EntryNotFoundError(message='API not supported.') From 138cb38d7874441073f3bffa298ce8a4d95a1cd5 Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Tue, 11 Mar 2025 11:13:10 +0800 Subject: [PATCH 34/34] add python_version in docker build workflow --- .github/workflows/docker-image.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 13f61ff3..e028d36a 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -26,8 +26,12 @@ on: other_params: description: 'Other params in --xxx xxx' required: false + python_version: + description: 'Python version to use, default is 3.10.14' + required: false + default: '3.10.14' -run-name: Docker-${{ inputs.modelscope_branch }}-${{ inputs.image_type }}-${{ inputs.workflow_name }}-by-@${{ github.actor }} +run-name: Docker-${{ inputs.modelscope_branch }}-${{ inputs.image_type }}-${{ inputs.workflow_name }}-${{ inputs.python_version }}-by-@${{ github.actor }} jobs: build: @@ -51,4 +55,4 @@ jobs: run: | set -e source ~/.bashrc - python docker/build_image.py --image_type ${{ github.event.inputs.image_type }} --modelscope_branch ${{ github.event.inputs.modelscope_branch }} --modelscope_version ${{ github.event.inputs.modelscope_version }} --swift_branch ${{ github.event.inputs.swift_branch }} --ci_image ${{ github.event.inputs.ci_image }} ${{ github.event.inputs.other_params }} + python docker/build_image.py --image_type ${{ github.event.inputs.image_type }} --python_version ${{ github.event.inputs.python_version }} --modelscope_branch ${{ github.event.inputs.modelscope_branch }} --modelscope_version ${{ github.event.inputs.modelscope_version }} --swift_branch ${{ github.event.inputs.swift_branch }} --ci_image ${{ github.event.inputs.ci_image }} ${{ github.event.inputs.other_params }}