From eaf4ea8f2198f5c940484c088efde99ac19b2d08 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Mon, 27 Jan 2025 21:46:34 +0800 Subject: [PATCH] fix comments --- modelscope/hub/api.py | 40 ++++++++++++++---- modelscope/hub/create_model.py | 65 ----------------------------- modelscope/hub/push_to_hub.py | 7 +++- modelscope/utils/hf_util/patcher.py | 18 ++++---- 4 files changed, 49 insertions(+), 81 deletions(-) delete mode 100644 modelscope/hub/create_model.py diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 63f954d6..694c56f5 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -9,6 +9,7 @@ import pickle import platform import re import shutil +import tempfile import uuid from collections import defaultdict from http import HTTPStatus @@ -47,7 +48,9 @@ from modelscope.hub.errors import (InvalidParameter, NotExistError, raise_for_http_status, raise_on_error) from modelscope.hub.git import GitCommandWrapper from modelscope.hub.repository import Repository -from modelscope.hub.utils.utils import (get_endpoint, get_readable_folder_size, +from modelscope.hub.utils.utils import (add_patterns_to_file, + add_patterns_to_gitattributes, + get_endpoint, get_readable_folder_size, get_release_datetime, model_id_to_group_owner_name) from modelscope.utils.constant import (DEFAULT_DATASET_REVISION, @@ -158,6 +161,7 @@ class HubApi: self.login(access_token) return True except AssertionError: + logger.warning('Login failed.') return False def create_model(self, @@ -1210,21 +1214,22 @@ class HubApi: repo_type: Optional[str] = REPO_TYPE_MODEL, chinese_name: Optional[str] = '', license: Optional[str] = Licenses.APACHE_V2, + **kwargs, ) -> str: # TODO: exist_ok - if not repo_id: raise ValueError('Repo id cannot be empty!') - if token: - self.login(access_token=token) - else: - logger.warning('No token provided, will use the cached token.') + self.try_login(token) + if '/' not in repo_id: + user_name = ModelScopeConfig.get_user_info()[0] + assert isinstance(user_name, str) + repo_id = f'{user_name}/{repo_id}' + logger.info( + f"'/' not in hub_model_id, pushing to personal repo {repo_id}") repo_id_list = repo_id.split('/') - if len(repo_id_list) != 2: - raise ValueError('Invalid repo id, should be in the format of `owner_name/repo_name`') namespace, repo_name = repo_id_list if repo_type == REPO_TYPE_MODEL: @@ -1240,6 +1245,25 @@ class HubApi: chinese_name=chinese_name, ) + with tempfile.TemporaryDirectory() as temp_cache_dir: + from modelscope.hub.repository import Repository + repo = Repository(temp_cache_dir, repo_id) + add_patterns_to_gitattributes( + repo, ['*.safetensors', '*.bin', '*.pt', '*.gguf']) + default_config = { + 'framework': 'pytorch', + 'task': 'text-generation', + 'allow_remote': True + } + config_json = kwargs.get('config_json') + if not config_json: + config_json = {} + config = {**default_config, **config_json} + add_patterns_to_file( + repo, + 'configuration.json', [json.dumps(config)], + ignore_push_error=True) + elif repo_type == REPO_TYPE_DATASET: visibilities = {k: v for k, v in DatasetVisibility.__dict__.items() if not k.startswith('__')} visibility: int = visibilities.get(visibility.upper()) diff --git a/modelscope/hub/create_model.py b/modelscope/hub/create_model.py deleted file mode 100644 index b1811acc..00000000 --- a/modelscope/hub/create_model.py +++ /dev/null @@ -1,65 +0,0 @@ -import tempfile -from typing import Any, Dict, Optional - -import json -from requests.exceptions import HTTPError - -from modelscope.hub.api import HubApi, ModelScopeConfig -from modelscope.hub.constants import ModelVisibility -from modelscope.utils.logger import get_logger -from .utils.utils import add_patterns_to_file, add_patterns_to_gitattributes - -logger = get_logger() - - -def create_model_repo(repo_id: str, - token: Optional[str] = None, - private: bool = False, - config_json: Optional[Dict[str, Any]] = None) -> str: - """Create model repo and create .gitattributes file and .gitignore file - - Args: - repo_id(str): The repo id - token(str, Optional): The access token of the user - private(bool): If is a private repo, default False - config_json(Dict[str, Any]): An optional config_json to fill into the configuration.json file, - If None, the default content will be uploaded: - ```json - {"framework": "pytorch", "task": "text-generation", "allow_remote": True} - ``` - You can manually modify this in the modelhub. - """ - api = HubApi() - assert repo_id is not None, 'Please enter a valid repo id' - api.try_login(token) - visibility = ModelVisibility.PRIVATE if private else ModelVisibility.PUBLIC - if '/' not in repo_id: - user_name = ModelScopeConfig.get_user_info()[0] - assert isinstance(user_name, str) - repo_id = f'{user_name}/{repo_id}' - logger.info( - f"'/' not in hub_model_id, pushing to personal repo {repo_id}") - try: - api.create_model(repo_id, visibility) - except HTTPError: - # The remote repository has been created - pass - - with tempfile.TemporaryDirectory() as temp_cache_dir: - from modelscope.hub.repository import Repository - repo = Repository(temp_cache_dir, repo_id) - add_patterns_to_gitattributes( - repo, ['*.safetensors', '*.bin', '*.pt', '*.gguf']) - default_config = { - 'framework': 'pytorch', - 'task': 'text-generation', - 'allow_remote': True - } - if not config_json: - config_json = {} - config = {**default_config, **config_json} - add_patterns_to_file( - repo, - 'configuration.json', [json.dumps(config)], - ignore_push_error=True) - return repo_id diff --git a/modelscope/hub/push_to_hub.py b/modelscope/hub/push_to_hub.py index fdd4a17f..3dc70b1d 100644 --- a/modelscope/hub/push_to_hub.py +++ b/modelscope/hub/push_to_hub.py @@ -24,7 +24,7 @@ _tasks = dict() _manager = None -def push_files_to_hub( +def _push_files_to_hub( path_or_fileobj: Union[str, Path], path_in_repo: str, repo_id: str, @@ -33,6 +33,11 @@ def push_files_to_hub( commit_message: Optional[str] = None, commit_description: Optional[str] = None, ): + """Push files to model hub incrementally + + This function if used for patch_hub, user is not recommended to call this. + This function will be merged to push_to_hub in later sprints. + """ if not os.path.exists(path_or_fileobj): return diff --git a/modelscope/utils/hf_util/patcher.py b/modelscope/utils/hf_util/patcher.py index a51f8911..93d0af1b 100644 --- a/modelscope/utils/hf_util/patcher.py +++ b/modelscope/utils/hf_util/patcher.py @@ -383,8 +383,12 @@ def _patch_hub(): Returns: RepoUrl: The URL of the created repository. """ - from modelscope.hub.create_model import create_model_repo - hub_model_id = create_model_repo(repo_id, token, private) + from modelscope.hub.api import HubApi + api = HubApi() + from modelscope.hub.constants import ModelVisibility + visibility = ModelVisibility.PRIVATE if private else ModelVisibility.PUBLIC + hub_model_id = api.create_repo( + repo_id, token=token, visibility=visibility, **kwargs) from huggingface_hub import RepoUrl return RepoUrl(url=hub_model_id, ) @@ -402,8 +406,8 @@ def _patch_hub(): ignore_patterns: Optional[Union[List[str], str]] = None, **kwargs, ): - from modelscope.hub.push_to_hub import push_files_to_hub - push_files_to_hub( + from modelscope.hub.push_to_hub import _push_files_to_hub + _push_files_to_hub( path_or_fileobj=folder_path, path_in_repo=path_in_repo, repo_id=repo_id, @@ -434,9 +438,9 @@ def _patch_hub(): commit_description: Optional[str] = None, **kwargs, ): - from modelscope.hub.push_to_hub import push_files_to_hub - push_files_to_hub(path_or_fileobj, path_in_repo, repo_id, token, - revision, commit_message, commit_description) + from modelscope.hub.push_to_hub import _push_files_to_hub + _push_files_to_hub(path_or_fileobj, path_in_repo, repo_id, token, + revision, commit_message, commit_description) # Patch repocard.validate from huggingface_hub import repocard