Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2026-09-01 19:49:03 +02:00
Code Issues Packages Projects Releases Wiki Activity

[Fix] Add endpoint for creating model repo (#1699)

Browse Source
This commit is contained in:
Xingjun.Wang
2026-04-29 01:37:24 +08:00
committed by GitHub
parent b3808b7c20
commit f0ba7bf885
3 changed files with 86 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
modelscope/hub/api.py
View File

@@ -949,13 +949,14 @@ class HubApi:
license=license,
chinese_name=chinese_name,
original_model_id=original_model_id,
token=token)
token=token,
endpoint=self.endpoint)
tmp_dir = os.path.join(model_dir, TEMPORARY_FOLDER_NAME) # make temporary folder
git_wrapper = GitCommandWrapper()
logger.info(f'Pushing folder {model_dir} as model {model_id}.')
logger.info(f'Total folder size {folder_size}, this may take a while depending on actual pushing size...')
try:
repo = Repository(model_dir=tmp_dir, clone_from=model_id, auth_token=token)
repo = Repository(model_dir=tmp_dir, clone_from=model_id, auth_token=token, endpoint=self.endpoint)
branches = git_wrapper.get_remote_branches(tmp_dir)
if revision not in branches:
logger.info(f'Creating new branch {revision}')
@@ -2216,11 +2217,12 @@ class HubApi:
chinese_name=chinese_name,
aigc_model=aigc_model,
token=token,
endpoint=endpoint,
)
if create_default_config:
with tempfile.TemporaryDirectory() as temp_cache_dir:
from modelscope.hub.repository import Repository
repo = Repository(temp_cache_dir, repo_id, auth_token=token)
repo = Repository(temp_cache_dir, repo_id, auth_token=token, endpoint=endpoint)
default_config = {
'framework': 'pytorch',
'task': 'text-generation',
@@ -2249,6 +2251,7 @@ class HubApi:
license=license,
visibility=visibility,
token=token,
endpoint=endpoint,
)
print(f'New dataset created successfully at {repo_url}.', flush=True)

77
modelscope/hub/git.py
View File

@@ -3,6 +3,7 @@
import os
import subprocess
from typing import List, Optional
from urllib.parse import urlparse, urlunparse
from modelscope.utils.logger import get_logger
from ..utils.constant import MASTER_MODEL_BRANCH
@@ -57,8 +58,8 @@ class GitCommandWrapper(metaclass=Singleton):
response.check_returncode()
return response
except subprocess.CalledProcessError as error:
std_out = response.stdout.decode('utf8')
std_err = error.stderr.decode('utf8')
std_out = response.stdout.decode('utf-8', errors='replace')
std_err = error.stderr.decode('utf-8', errors='replace')
if 'nothing to commit' in std_out:
logger.info(
'Nothing to commit, your local repo is upto date with remote'
@@ -80,10 +81,51 @@ class GitCommandWrapper(metaclass=Singleton):
logger.debug(rsp.stdout.decode('utf8'))
def _add_token(self, token: str, url: str):
if token:
if '//oauth2' not in url:
url = url.replace('//', '//oauth2:%s@' % token)
return url
"""Inject OAuth2 token into an HTTP(S) git URL.
Uses ``urllib.parse`` for reliable URL component handling,
avoiding naive string replacement that can corrupt URLs
containing multiple ``://`` sequences.
Args:
token: OAuth2 access token.
url: Remote URL (HTTP, HTTPS, or SSH).
Returns:
URL with ``oauth2:<token>@`` injected into the *netloc*,
or the original *url* unchanged when:
* *token* is falsy,
* the URL already carries an ``oauth2`` credential,
* the scheme is not HTTP/HTTPS (e.g. ``ssh://``, ``git@``).
"""
if not token:
return url
# SSH URLs authenticate via keys, not tokens.
if url.startswith('git@'):
return url
try:
parsed = urlparse(url)
except Exception:
return url
# Only inject into HTTP(S) URLs.
if parsed.scheme not in ('http', 'https'):
return url
# Prevent double injection.
if parsed.username == 'oauth2':
return url
# Reconstruct netloc: oauth2:<token>@host[:port]
host = parsed.hostname or ''
if parsed.port:
host = f'{host}:{parsed.port}'
netloc = f'oauth2:{token}@{host}'
return urlunparse(parsed._replace(netloc=netloc))
def remove_token_from_url(self, url: str):
if url and '//oauth2' in url:
@@ -101,7 +143,7 @@ class GitCommandWrapper(metaclass=Singleton):
return False
def git_lfs_install(self, repo_dir):
cmd = ['-C', repo_dir, 'lfs', 'install']
cmd = ['-C', repo_dir, 'lfs', 'install', '--force']
try:
self._run_git_command(*cmd)
return True
@@ -135,9 +177,24 @@ class GitCommandWrapper(metaclass=Singleton):
clone_args = '-C %s clone %s' % (repo_base_dir, url)
logger.debug(clone_args)
clone_args = clone_args.split(' ')
response = self._run_git_command(*clone_args)
logger.debug(response.stdout.decode('utf8'))
return response
try:
response = self._run_git_command(*clone_args)
logger.debug(response.stdout.decode('utf8'))
return response
except GitError:
# git clone may succeed but still exit non-zero when an
# external hook (e.g. a custom core.hooksPath that wraps
# ``git lfs post-merge``) returns a non-zero code. When the
# repository was actually cloned, treat this as a warning.
repo_dir = os.path.join(repo_base_dir, repo_name)
if os.path.isdir(os.path.join(repo_dir, '.git')):
logger.warning(
'git clone exited with non-zero status but the '
'repository was cloned successfully at %s. '
'This is usually caused by a post-clone hook '
'(e.g. core.hooksPath). Continuing.', repo_dir)
return None
raise
def add_user_info(self, repo_base_dir, repo_name):
from modelscope.hub.api import ModelScopeConfig

18
modelscope/hub/repository.py
View File

@@ -24,7 +24,8 @@ class Repository:
clone_from: str,
revision: Optional[str] = DEFAULT_REPOSITORY_REVISION,
auth_token: Optional[str] = None,
git_path: Optional[str] = None):
git_path: Optional[str] = None,
endpoint: Optional[str] = None):
"""Instantiate a Repository object by cloning the remote ModelScopeHub repo
Args:
@@ -36,10 +37,12 @@ class Repository:
Usually you can safely ignore the parameter as the token is already
saved when you login the first time, if None, we will use saved token.
git_path (str, optional): The git command line path, if None, we use 'git'
endpoint (str, optional): The ModelScope endpoint URL. If None, use default endpoint.
Raises:
InvalidParameter: revision is None.
"""
self._endpoint = endpoint
self.model_dir = model_dir
self.model_base_dir = os.path.dirname(model_dir)
self.model_repo_name = os.path.basename(model_dir)
@@ -70,7 +73,7 @@ class Repository:
self.model_repo_name, revision)
if git_wrapper.is_lfs_installed():
git_wrapper.git_lfs_install(self.model_dir) # init repo lfs
git_wrapper.git_lfs_install(self.model_dir)
# add user info if login
self.git_wrapper.add_user_info(self.model_base_dir,
@@ -79,7 +82,8 @@ class Repository:
self.git_wrapper.config_auth_token(self.model_dir, self.auth_token)
def _get_model_id_url(self, model_id):
url = f'{get_endpoint()}/{model_id}.git'
endpoint = self._endpoint if self._endpoint else get_endpoint()
url = f'{endpoint}/{model_id}.git'
return url
def _get_remote_url(self):
@@ -207,7 +211,8 @@ class DatasetRepository:
dataset_id: str,
revision: Optional[str] = DEFAULT_DATASET_REVISION,
auth_token: Optional[str] = None,
git_path: Optional[str] = None):
git_path: Optional[str] = None,
endpoint: Optional[str] = None):
"""
Instantiate a Dataset Repository object by cloning the remote ModelScope dataset repo
@@ -220,10 +225,12 @@ class DatasetRepository:
Usually you can safely ignore the parameter as the token is
already saved when you login the first time, if None, we will use saved token.
git_path (str, optional): The git command line path, if None, we use 'git'
endpoint (str, optional): The ModelScope endpoint URL. If None, use default endpoint.
Raises:
InvalidParameter: parameter invalid.
"""
self._endpoint = endpoint
self.dataset_id = dataset_id
if not repo_work_dir or not isinstance(repo_work_dir, str):
err_msg = 'dataset_work_dir must be provided!'
@@ -314,7 +321,8 @@ class DatasetRepository:
remote_branch=branch)
def _get_repo_url(self, dataset_id):
return f'{get_endpoint()}/datasets/{dataset_id}.git'
endpoint = self._endpoint if self._endpoint else get_endpoint()
return f'{endpoint}/datasets/{dataset_id}.git'
def _get_remote_url(self):
try: