add repo existence check api

This commit is contained in:
Yingda Chen
2024-11-01 18:06:04 +08:00
parent 250b72fce7
commit e2b55397f9
4 changed files with 62 additions and 8 deletions

View File

@@ -248,6 +248,48 @@ class HubApi:
else:
raise_for_http_status(r)
def repo_exists(
self,
repo_id: str,
*,
repo_type: Optional[str] = None,
) -> bool:
"""
Checks if a repository exists on ModelScope
Args:
repo_id (`str`):
A namespace (user or an organization) and a repo name separated
by a `/`.
repo_type (`str`, *optional*):
`None` or `"model"` if getting repository info from a model. Default is `None`.
TODO: support dataset and studio
Returns:
True if the repository exists, False otherwise.
"""
if (repo_type is not None) and repo_type.lower != 'model':
raise Exception('Not support repo-type: %s' % repo_type)
if (repo_id is None) or repo_id.count('/') != 1:
raise Exception('Invalid repo_id: %s, must be of format namespace/name' % repo_type)
cookies = ModelScopeConfig.get_cookies()
owner_or_group, name = model_id_to_group_owner_name(repo_id)
path = f'{self.endpoint}/api/v1/models/{owner_or_group}/{name}'
r = self.session.get(path, cookies=cookies,
headers=self.builder_headers(self.headers))
code = handle_http_response(r, logger, cookies, repo_id, False)
logger.info(f'check repo_exists status code {code}.')
if code == 200:
return True
elif code == 404:
return False
else:
raise Exception(
'Failed to check existence of repo: %s, make sure you have access authorization.'
% repo_type)
def push_model(self,
model_id: str,
model_dir: str,

View File

@@ -2,6 +2,7 @@
import logging
from http import HTTPStatus
from typing import Optional
import requests
from requests.exceptions import HTTPError
@@ -86,8 +87,11 @@ def handle_http_post_error(response, url, request_body):
(url, request_body, message, get_request_id(response))) from error
def handle_http_response(response: requests.Response, logger, cookies,
model_id):
def handle_http_response(response: requests.Response,
logger,
cookies,
model_id,
raise_on_error: Optional[bool] = True) -> int:
http_error_msg = ''
if isinstance(response.reason, bytes):
try:
@@ -113,9 +117,11 @@ def handle_http_response(response: requests.Response, logger, cookies,
elif 500 <= response.status_code < 600:
http_error_msg = u'%s Server Error: %s, Request id: %s, for url: %s' % (
response.status_code, reason, request_id, response.url)
if http_error_msg: # there is error.
if http_error_msg and raise_on_error: # there is error.
logger.error(http_error_msg)
raise HTTPError(http_error_msg, response=response)
else:
return response.status_code
def raise_on_error(rsp):

View File

@@ -6,7 +6,7 @@ from modelscope.hub.api import HubApi
from modelscope.utils.hub import create_model_if_not_exist
# note this is temporary before official account management is ready
YOUR_ACCESS_TOKEN = 'token'
YOUR_ACCESS_TOKEN = 'Get SDK token from https://www.modelscope.cn/my/myaccesstoken'
class HubExampleTest(unittest.TestCase):
@@ -18,10 +18,10 @@ class HubExampleTest(unittest.TestCase):
@unittest.skip('to be used for local test only')
def test_example_model_creation(self):
# ATTENTION:change to proper model names before use
model_name = 'cv_unet_person-image-cartoon_compound-models'
model_chinese_name = '达摩卡通化模型'
model_org = 'damo'
model_id = '%s/%s' % (model_org, model_name)
model_name = 'model-name'
model_chinese_name = '我的测试模型'
model_owner = 'iic'
model_id = '%s/%s' % (model_owner, model_name)
created = create_model_if_not_exist(self.api, model_id,
model_chinese_name)
if not created:

View File

@@ -47,6 +47,12 @@ class HubUploadTest(unittest.TestCase):
except Exception:
pass
def test_repo_exist(self):
res = self.api.repo_exists('Qwen/Qwen2.5-7B-Instruct')
self.assertTrue(res)
res = self.api.repo_exists('Qwen/not-a-repo')
self.assertFalse(res)
def test_upload_exits_repo_master(self):
logger.info('basic test for upload!')
self.api.login(TEST_ACCESS_TOKEN1)