Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2026-09-01 19:49:03 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix comments

Browse Source
This commit is contained in:
yuze.zyz
2025-01-27 21:46:34 +08:00
parent 565e45370e
commit eaf4ea8f21
4 changed files with 49 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
modelscope/hub/api.py
View File

@@ -9,6 +9,7 @@ import pickle
import platform
import re
import shutil
import tempfile
import uuid
from collections import defaultdict
from http import HTTPStatus
@@ -47,7 +48,9 @@ from modelscope.hub.errors import (InvalidParameter, NotExistError,
raise_for_http_status, raise_on_error)
from modelscope.hub.git import GitCommandWrapper
from modelscope.hub.repository import Repository
from modelscope.hub.utils.utils import (get_endpoint, get_readable_folder_size,
from modelscope.hub.utils.utils import (add_patterns_to_file,
add_patterns_to_gitattributes,
get_endpoint, get_readable_folder_size,
get_release_datetime,
model_id_to_group_owner_name)
from modelscope.utils.constant import (DEFAULT_DATASET_REVISION,
@@ -158,6 +161,7 @@ class HubApi:
self.login(access_token)
return True
except AssertionError:
logger.warning('Login failed.')
return False
def create_model(self,
@@ -1210,21 +1214,22 @@ class HubApi:
repo_type: Optional[str] = REPO_TYPE_MODEL,
chinese_name: Optional[str] = '',
license: Optional[str] = Licenses.APACHE_V2,
**kwargs,
) -> str:
# TODO: exist_ok
if not repo_id:
raise ValueError('Repo id cannot be empty!')
if token:
self.login(access_token=token)
else:
logger.warning('No token provided, will use the cached token.')
self.try_login(token)
if '/' not in repo_id:
user_name = ModelScopeConfig.get_user_info()[0]
assert isinstance(user_name, str)
repo_id = f'{user_name}/{repo_id}'
logger.info(
f"'/' not in hub_model_id, pushing to personal repo {repo_id}")
repo_id_list = repo_id.split('/')
if len(repo_id_list) != 2:
raise ValueError('Invalid repo id, should be in the format of `owner_name/repo_name`')
namespace, repo_name = repo_id_list
if repo_type == REPO_TYPE_MODEL:
@@ -1240,6 +1245,25 @@ class HubApi:
chinese_name=chinese_name,
)
with tempfile.TemporaryDirectory() as temp_cache_dir:
from modelscope.hub.repository import Repository
repo = Repository(temp_cache_dir, repo_id)
add_patterns_to_gitattributes(
repo, ['*.safetensors', '*.bin', '*.pt', '*.gguf'])
default_config = {
'framework': 'pytorch',
'task': 'text-generation',
'allow_remote': True
}
config_json = kwargs.get('config_json')
if not config_json:
config_json = {}
config = {**default_config, **config_json}
add_patterns_to_file(
repo,
'configuration.json', [json.dumps(config)],
ignore_push_error=True)
elif repo_type == REPO_TYPE_DATASET:
visibilities = {k: v for k, v in DatasetVisibility.__dict__.items() if not k.startswith('__')}
visibility: int = visibilities.get(visibility.upper())

65
modelscope/hub/create_model.py
View File

@@ -1,65 +0,0 @@
import tempfile
from typing import Any, Dict, Optional
import json
from requests.exceptions import HTTPError
from modelscope.hub.api import HubApi, ModelScopeConfig
from modelscope.hub.constants import ModelVisibility
from modelscope.utils.logger import get_logger
from .utils.utils import add_patterns_to_file, add_patterns_to_gitattributes
logger = get_logger()
def create_model_repo(repo_id: str,
token: Optional[str] = None,
private: bool = False,
config_json: Optional[Dict[str, Any]] = None) -> str:
"""Create model repo and create .gitattributes file and .gitignore file
Args:
repo_id(str): The repo id
token(str, Optional): The access token of the user
private(bool): If is a private repo, default False
config_json(Dict[str, Any]): An optional config_json to fill into the configuration.json file,
If None, the default content will be uploaded:
```json
{"framework": "pytorch", "task": "text-generation", "allow_remote": True}
```
You can manually modify this in the modelhub.
"""
api = HubApi()
assert repo_id is not None, 'Please enter a valid repo id'
api.try_login(token)
visibility = ModelVisibility.PRIVATE if private else ModelVisibility.PUBLIC
if '/' not in repo_id:
user_name = ModelScopeConfig.get_user_info()[0]
assert isinstance(user_name, str)
repo_id = f'{user_name}/{repo_id}'
logger.info(
f"'/' not in hub_model_id, pushing to personal repo {repo_id}")
try:
api.create_model(repo_id, visibility)
except HTTPError:
# The remote repository has been created
pass
with tempfile.TemporaryDirectory() as temp_cache_dir:
from modelscope.hub.repository import Repository
repo = Repository(temp_cache_dir, repo_id)
add_patterns_to_gitattributes(
repo, ['*.safetensors', '*.bin', '*.pt', '*.gguf'])
default_config = {
'framework': 'pytorch',
'task': 'text-generation',
'allow_remote': True
}
if not config_json:
config_json = {}
config = {**default_config, **config_json}
add_patterns_to_file(
repo,
'configuration.json', [json.dumps(config)],
ignore_push_error=True)
return repo_id

7
modelscope/hub/push_to_hub.py
View File

@@ -24,7 +24,7 @@ _tasks = dict()
_manager = None
def push_files_to_hub(
def _push_files_to_hub(
path_or_fileobj: Union[str, Path],
path_in_repo: str,
repo_id: str,
@@ -33,6 +33,11 @@ def push_files_to_hub(
commit_message: Optional[str] = None,
commit_description: Optional[str] = None,
):
"""Push files to model hub incrementally
This function if used for patch_hub, user is not recommended to call this.
This function will be merged to push_to_hub in later sprints.
"""
if not os.path.exists(path_or_fileobj):
return

18
modelscope/utils/hf_util/patcher.py
View File

@@ -383,8 +383,12 @@ def _patch_hub():
Returns:
RepoUrl: The URL of the created repository.
"""
from modelscope.hub.create_model import create_model_repo
hub_model_id = create_model_repo(repo_id, token, private)
from modelscope.hub.api import HubApi
api = HubApi()
from modelscope.hub.constants import ModelVisibility
visibility = ModelVisibility.PRIVATE if private else ModelVisibility.PUBLIC
hub_model_id = api.create_repo(
repo_id, token=token, visibility=visibility, **kwargs)
from huggingface_hub import RepoUrl
return RepoUrl(url=hub_model_id, )
@@ -402,8 +406,8 @@ def _patch_hub():
ignore_patterns: Optional[Union[List[str], str]] = None,
**kwargs,
):
from modelscope.hub.push_to_hub import push_files_to_hub
push_files_to_hub(
from modelscope.hub.push_to_hub import _push_files_to_hub
_push_files_to_hub(
path_or_fileobj=folder_path,
path_in_repo=path_in_repo,
repo_id=repo_id,
@@ -434,9 +438,9 @@ def _patch_hub():
commit_description: Optional[str] = None,
**kwargs,
):
from modelscope.hub.push_to_hub import push_files_to_hub
push_files_to_hub(path_or_fileobj, path_in_repo, repo_id, token,
revision, commit_message, commit_description)
from modelscope.hub.push_to_hub import _push_files_to_hub
_push_files_to_hub(path_or_fileobj, path_in_repo, repo_id, token,
revision, commit_message, commit_description)
# Patch repocard.validate
from huggingface_hub import repocard