From f0ba7bf88583e86a4b5c9726c5126528fc223ee9 Mon Sep 17 00:00:00 2001 From: "Xingjun.Wang" Date: Wed, 29 Apr 2026 01:37:24 +0800 Subject: [PATCH] [Fix] Add endpoint for creating model repo (#1699) --- modelscope/hub/api.py | 9 +++-- modelscope/hub/git.py | 77 +++++++++++++++++++++++++++++++----- modelscope/hub/repository.py | 18 ++++++--- 3 files changed, 86 insertions(+), 18 deletions(-) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 990271b0..4e700ad7 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -949,13 +949,14 @@ class HubApi: license=license, chinese_name=chinese_name, original_model_id=original_model_id, - token=token) + token=token, + endpoint=self.endpoint) tmp_dir = os.path.join(model_dir, TEMPORARY_FOLDER_NAME) # make temporary folder git_wrapper = GitCommandWrapper() logger.info(f'Pushing folder {model_dir} as model {model_id}.') logger.info(f'Total folder size {folder_size}, this may take a while depending on actual pushing size...') try: - repo = Repository(model_dir=tmp_dir, clone_from=model_id, auth_token=token) + repo = Repository(model_dir=tmp_dir, clone_from=model_id, auth_token=token, endpoint=self.endpoint) branches = git_wrapper.get_remote_branches(tmp_dir) if revision not in branches: logger.info(f'Creating new branch {revision}') @@ -2216,11 +2217,12 @@ class HubApi: chinese_name=chinese_name, aigc_model=aigc_model, token=token, + endpoint=endpoint, ) if create_default_config: with tempfile.TemporaryDirectory() as temp_cache_dir: from modelscope.hub.repository import Repository - repo = Repository(temp_cache_dir, repo_id, auth_token=token) + repo = Repository(temp_cache_dir, repo_id, auth_token=token, endpoint=endpoint) default_config = { 'framework': 'pytorch', 'task': 'text-generation', @@ -2249,6 +2251,7 @@ class HubApi: license=license, visibility=visibility, token=token, + endpoint=endpoint, ) print(f'New dataset created successfully at {repo_url}.', flush=True) diff --git a/modelscope/hub/git.py b/modelscope/hub/git.py index d03ca773..64d37620 100644 --- a/modelscope/hub/git.py +++ b/modelscope/hub/git.py @@ -3,6 +3,7 @@ import os import subprocess from typing import List, Optional +from urllib.parse import urlparse, urlunparse from modelscope.utils.logger import get_logger from ..utils.constant import MASTER_MODEL_BRANCH @@ -57,8 +58,8 @@ class GitCommandWrapper(metaclass=Singleton): response.check_returncode() return response except subprocess.CalledProcessError as error: - std_out = response.stdout.decode('utf8') - std_err = error.stderr.decode('utf8') + std_out = response.stdout.decode('utf-8', errors='replace') + std_err = error.stderr.decode('utf-8', errors='replace') if 'nothing to commit' in std_out: logger.info( 'Nothing to commit, your local repo is upto date with remote' @@ -80,10 +81,51 @@ class GitCommandWrapper(metaclass=Singleton): logger.debug(rsp.stdout.decode('utf8')) def _add_token(self, token: str, url: str): - if token: - if '//oauth2' not in url: - url = url.replace('//', '//oauth2:%s@' % token) - return url + """Inject OAuth2 token into an HTTP(S) git URL. + + Uses ``urllib.parse`` for reliable URL component handling, + avoiding naive string replacement that can corrupt URLs + containing multiple ``://`` sequences. + + Args: + token: OAuth2 access token. + url: Remote URL (HTTP, HTTPS, or SSH). + + Returns: + URL with ``oauth2:@`` injected into the *netloc*, + or the original *url* unchanged when: + + * *token* is falsy, + * the URL already carries an ``oauth2`` credential, + * the scheme is not HTTP/HTTPS (e.g. ``ssh://``, ``git@``). + """ + if not token: + return url + + # SSH URLs authenticate via keys, not tokens. + if url.startswith('git@'): + return url + + try: + parsed = urlparse(url) + except Exception: + return url + + # Only inject into HTTP(S) URLs. + if parsed.scheme not in ('http', 'https'): + return url + + # Prevent double injection. + if parsed.username == 'oauth2': + return url + + # Reconstruct netloc: oauth2:@host[:port] + host = parsed.hostname or '' + if parsed.port: + host = f'{host}:{parsed.port}' + netloc = f'oauth2:{token}@{host}' + + return urlunparse(parsed._replace(netloc=netloc)) def remove_token_from_url(self, url: str): if url and '//oauth2' in url: @@ -101,7 +143,7 @@ class GitCommandWrapper(metaclass=Singleton): return False def git_lfs_install(self, repo_dir): - cmd = ['-C', repo_dir, 'lfs', 'install'] + cmd = ['-C', repo_dir, 'lfs', 'install', '--force'] try: self._run_git_command(*cmd) return True @@ -135,9 +177,24 @@ class GitCommandWrapper(metaclass=Singleton): clone_args = '-C %s clone %s' % (repo_base_dir, url) logger.debug(clone_args) clone_args = clone_args.split(' ') - response = self._run_git_command(*clone_args) - logger.debug(response.stdout.decode('utf8')) - return response + try: + response = self._run_git_command(*clone_args) + logger.debug(response.stdout.decode('utf8')) + return response + except GitError: + # git clone may succeed but still exit non-zero when an + # external hook (e.g. a custom core.hooksPath that wraps + # ``git lfs post-merge``) returns a non-zero code. When the + # repository was actually cloned, treat this as a warning. + repo_dir = os.path.join(repo_base_dir, repo_name) + if os.path.isdir(os.path.join(repo_dir, '.git')): + logger.warning( + 'git clone exited with non-zero status but the ' + 'repository was cloned successfully at %s. ' + 'This is usually caused by a post-clone hook ' + '(e.g. core.hooksPath). Continuing.', repo_dir) + return None + raise def add_user_info(self, repo_base_dir, repo_name): from modelscope.hub.api import ModelScopeConfig diff --git a/modelscope/hub/repository.py b/modelscope/hub/repository.py index 6519d8ea..1dd6665b 100644 --- a/modelscope/hub/repository.py +++ b/modelscope/hub/repository.py @@ -24,7 +24,8 @@ class Repository: clone_from: str, revision: Optional[str] = DEFAULT_REPOSITORY_REVISION, auth_token: Optional[str] = None, - git_path: Optional[str] = None): + git_path: Optional[str] = None, + endpoint: Optional[str] = None): """Instantiate a Repository object by cloning the remote ModelScopeHub repo Args: @@ -36,10 +37,12 @@ class Repository: Usually you can safely ignore the parameter as the token is already saved when you login the first time, if None, we will use saved token. git_path (str, optional): The git command line path, if None, we use 'git' + endpoint (str, optional): The ModelScope endpoint URL. If None, use default endpoint. Raises: InvalidParameter: revision is None. """ + self._endpoint = endpoint self.model_dir = model_dir self.model_base_dir = os.path.dirname(model_dir) self.model_repo_name = os.path.basename(model_dir) @@ -70,7 +73,7 @@ class Repository: self.model_repo_name, revision) if git_wrapper.is_lfs_installed(): - git_wrapper.git_lfs_install(self.model_dir) # init repo lfs + git_wrapper.git_lfs_install(self.model_dir) # add user info if login self.git_wrapper.add_user_info(self.model_base_dir, @@ -79,7 +82,8 @@ class Repository: self.git_wrapper.config_auth_token(self.model_dir, self.auth_token) def _get_model_id_url(self, model_id): - url = f'{get_endpoint()}/{model_id}.git' + endpoint = self._endpoint if self._endpoint else get_endpoint() + url = f'{endpoint}/{model_id}.git' return url def _get_remote_url(self): @@ -207,7 +211,8 @@ class DatasetRepository: dataset_id: str, revision: Optional[str] = DEFAULT_DATASET_REVISION, auth_token: Optional[str] = None, - git_path: Optional[str] = None): + git_path: Optional[str] = None, + endpoint: Optional[str] = None): """ Instantiate a Dataset Repository object by cloning the remote ModelScope dataset repo @@ -220,10 +225,12 @@ class DatasetRepository: Usually you can safely ignore the parameter as the token is already saved when you login the first time, if None, we will use saved token. git_path (str, optional): The git command line path, if None, we use 'git' + endpoint (str, optional): The ModelScope endpoint URL. If None, use default endpoint. Raises: InvalidParameter: parameter invalid. """ + self._endpoint = endpoint self.dataset_id = dataset_id if not repo_work_dir or not isinstance(repo_work_dir, str): err_msg = 'dataset_work_dir must be provided!' @@ -314,7 +321,8 @@ class DatasetRepository: remote_branch=branch) def _get_repo_url(self, dataset_id): - return f'{get_endpoint()}/datasets/{dataset_id}.git' + endpoint = self._endpoint if self._endpoint else get_endpoint() + return f'{endpoint}/datasets/{dataset_id}.git' def _get_remote_url(self): try: