Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2026-09-01 19:49:03 +02:00
Code Issues Packages Projects Releases Wiki Activity

damo to iic rename (#736)

Browse Source 
* add version time

* fix list model none issue

* damo to iic

* rmove unused import

* fix preprocess issue

* fix revison None bug

* fix bug

---------

Co-authored-by: mulin.lyh <mulin.lyh@taobao.com>
This commit is contained in:
liuyhwangyh
2024-01-29 09:38:23 +08:00
committed by GitHub
parent ade394d68c
commit 3d442b5c7e
4 changed files with 117 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
modelscope/hub/api.py
View File

@@ -429,6 +429,30 @@ class HubApi:
use_cookies: Union[bool, CookieJar] = False) -> List[str]:
"""Get model branch and tags.
Args:
model_id (str): The model id
cutoff_timestamp (int): Tags created before the cutoff will be included.
The timestamp is represented by the seconds elapsed from the epoch time.
use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
will load cookie from local. Defaults to False.
Returns:
Tuple[List[str], List[str]]: Return list of branch name and tags
"""
tags_details = self.list_model_revisions_detail(model_id=model_id,
cutoff_timestamp=cutoff_timestamp,
use_cookies=use_cookies)
tags = [x['Revision'] for x in tags_details
] if tags_details else []
return tags
def list_model_revisions_detail(
self,
model_id: str,
cutoff_timestamp: Optional[int] = None,
use_cookies: Union[bool, CookieJar] = False) -> List[str]:
"""Get model branch and tags.
Args:
model_id (str): The model id
cutoff_timestamp (int): Tags created before the cutoff will be included.
@@ -450,66 +474,84 @@ class HubApi:
raise_on_error(d)
info = d[API_RESPONSE_FIELD_DATA]
# tags returned from backend are guaranteed to be ordered by create-time
tags = [x['Revision'] for x in info['RevisionMap']['Tags']
] if info['RevisionMap']['Tags'] else []
return tags
return info['RevisionMap']['Tags']
def get_valid_revision(self,
model_id: str,
revision=None,
cookies: Optional[CookieJar] = None):
def get_branch_tag_detail(self, details, name):
for item in details:
if item['Revision'] == name:
return item
return None
def get_valid_revision_detail(self,
model_id: str,
revision=None,
cookies: Optional[CookieJar] = None):
release_timestamp = get_release_datetime()
current_timestamp = int(round(datetime.datetime.now().timestamp()))
# for active development in library codes (non-release-branches), release_timestamp
# is set to be a far-away-time-in-the-future, to ensure that we shall
# get the master-HEAD version from model repo by default (when no revision is provided)
all_branches_detail, all_tags_detail = self.get_model_branches_and_tags_details(
model_id, use_cookies=False if cookies is None else cookies)
all_branches = [x['Revision'] for x in all_branches_detail] if all_branches_detail else []
all_tags = [x['Revision'] for x in all_tags_detail] if all_tags_detail else []
if release_timestamp > current_timestamp + ONE_YEAR_SECONDS:
branches, tags = self.get_model_branches_and_tags(
model_id, use_cookies=False if cookies is None else cookies)
if revision is None:
revision = MASTER_MODEL_BRANCH
logger.info(
'Model revision not specified, use default: %s in development mode'
% revision)
if revision not in branches and revision not in tags:
if revision not in all_branches and revision not in all_tags:
raise NotExistError('The model: %s has no revision : %s .' % (model_id, revision))
revision_detail = self.get_branch_tag_detail(all_tags_detail, revision)
if revision_detail is None:
revision_detail = self.get_branch_tag_detail(all_branches_detail, revision)
logger.info('Development mode use revision: %s' % revision)
else:
all_revisions = self.list_model_revisions(
model_id,
cutoff_timestamp=current_timestamp,
use_cookies=False if cookies is None else cookies)
if len(all_revisions) == 0:
if len(all_tags_detail) == 0: # use no revision use master as default.
if revision is None or revision == MASTER_MODEL_BRANCH:
revision = MASTER_MODEL_BRANCH
else:
raise NotExistError('The model: %s has no revision: %s !' % (model_id, revision))
revision_detail = self.get_branch_tag_detail(all_branches_detail, revision)
else:
if revision is None: # user not specified revision, use latest revision before release time
revisions = self.list_model_revisions(
model_id,
cutoff_timestamp=release_timestamp,
use_cookies=False if cookies is None else cookies)
if len(revisions) > 0:
revision = revisions[0] # use latest revision before release time.
revisions_detail = [x for x in
all_tags_detail if x['CreatedAt'] <= release_timestamp] if all_tags_detail else [] # noqa E501
if len(revisions_detail) > 0:
revision = revisions_detail[0]['Revision'] # use latest revision before release time.
revision_detail = revisions_detail[0]
else:
revision = MASTER_MODEL_BRANCH
vl = '[%s]' % ','.join(all_revisions)
revision_detail = self.get_branch_tag_detail(all_branches_detail, revision)
vl = '[%s]' % ','.join(all_tags)
logger.warning('Model revision should be specified from revisions: %s' % (vl))
logger.warning('Model revision not specified, use revision: %s' % revision)
else:
# use user-specified revision
if revision not in all_revisions:
if revision not in all_tags:
if revision == MASTER_MODEL_BRANCH:
logger.warning('Using the master branch is fragile, please use it with caution!')
revision_detail = self.get_branch_tag_detail(all_branches_detail, revision)
else:
vl = '[%s]' % ','.join(all_revisions)
vl = '[%s]' % ','.join(all_tags)
raise NotExistError('The model: %s has no revision: %s valid are: %s!' %
(model_id, revision, vl))
else:
revision_detail = self.get_branch_tag_detail(all_tags_detail, revision)
logger.info('Use user-specified model revision: %s' % revision)
return revision
return revision_detail
def get_model_branches_and_tags(
def get_valid_revision(self,
model_id: str,
revision=None,
cookies: Optional[CookieJar] = None):
return self.get_valid_revision_detail(model_id=model_id,
revision=revision,
cookies=cookies)['Revision']
def get_model_branches_and_tags_details(
self,
model_id: str,
use_cookies: Union[bool, CookieJar] = False,
@@ -533,10 +575,29 @@ class HubApi:
d = r.json()
raise_on_error(d)
info = d[API_RESPONSE_FIELD_DATA]
branches = [x['Revision'] for x in info['RevisionMap']['Branches']
] if info['RevisionMap']['Branches'] else []
tags = [x['Revision'] for x in info['RevisionMap']['Tags']
] if info['RevisionMap']['Tags'] else []
return info['RevisionMap']['Branches'], info['RevisionMap']['Tags']
def get_model_branches_and_tags(
self,
model_id: str,
use_cookies: Union[bool, CookieJar] = False,
) -> Tuple[List[str], List[str]]:
"""Get model branch and tags.
Args:
model_id (str): The model id
use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
will load cookie from local. Defaults to False.
Returns:
Tuple[List[str], List[str]]: Return list of branch name and tags
"""
branches_detail, tags_detail = self.get_model_branches_and_tags_details(model_id=model_id,
use_cookies=use_cookies)
branches = [x['Revision'] for x in branches_detail
] if branches_detail else []
tags = [x['Revision'] for x in tags_detail
] if tags_detail else []
return branches, tags
def get_model_files(self,

5
modelscope/hub/snapshot_download.py
View File

@@ -94,8 +94,9 @@ def snapshot_download(model_id: str,
_api = HubApi()
if cookies is None:
cookies = ModelScopeConfig.get_cookies()
revision = _api.get_valid_revision(
revision_detail = _api.get_valid_revision_detail(
model_id, revision=revision, cookies=cookies)
revision = revision_detail['Revision']
snapshot_header = headers if 'CI_TEST' in os.environ else {
**headers,
@@ -165,6 +166,6 @@ def snapshot_download(model_id: str,
# put file into to cache
cache.put_file(model_file, temp_file)
cache.save_model_version(revision=revision)
cache.save_model_version(revision_info=revision_detail)
return os.path.join(cache.get_root_location())

7
modelscope/hub/utils/caching.py
View File

@@ -5,6 +5,7 @@ import os
import pickle
import tempfile
from shutil import move, rmtree
from typing import Dict
from modelscope.utils.logger import get_logger
@@ -159,11 +160,13 @@ class ModelFileSystemCache(FileSystemCache):
else:
return None
def save_model_version(self, revision: str):
def save_model_version(self, revision_info: Dict):
model_version_file_path = os.path.join(
self.cache_root_location, FileSystemCache.MODEL_VERSION_FILE_NAME)
with open(model_version_file_path, 'w') as f:
f.write(revision)
version_info_str = 'Revision:%s,CreatedAt:%s' % (
revision_info['Revision'], revision_info['CreatedAt'])
f.write(version_info_str)
def get_model_id(self):
return self.model_meta[FileSystemCache.MODEL_META_MODEL_ID]

26
tests/run_analysis.py
View File

@@ -2,7 +2,6 @@
import os
import subprocess
import sys
from fnmatch import fnmatch
from trainers.model_trainer_map import model_trainer_map
@@ -12,9 +11,9 @@ from utils.source_file_analyzer import (get_all_register_modules,
get_import_map)
from modelscope.hub.api import HubApi
from modelscope.hub.errors import NotExistError
from modelscope.hub.file_download import model_file_download
from modelscope.hub.utils.utils import get_cache_dir
from modelscope.hub.utils.utils import (get_cache_dir,
model_id_to_group_owner_name)
from modelscope.utils.config import Config
from modelscope.utils.constant import ModelFile
from modelscope.utils.logger import get_logger
@@ -27,10 +26,14 @@ def get_models_info(groups: list) -> dict:
api = HubApi()
for group in groups:
page = 1
total_count = 0
while True:
query_result = api.list_models(group, page, 100)
models.extend(query_result['Models'])
if len(models) >= query_result['TotalCount']:
if query_result['Models'] is not None:
models.extend(query_result['Models'])
elif total_count != 0:
total_count = query_result['TotalCount']
if len(models) >= total_count:
break
page += 1
cache_root = get_cache_dir()
@@ -218,7 +221,12 @@ def get_test_suites_to_run():
all_register_modules)
# task_pipeline_test_suite_map key: pipeline task, value: case file path
# trainer_test_suite_map key: trainer_name, value: case file path
models_info = get_models_info(['damo'])
iic_models_info = get_models_info(['iic'])
models_info = {}
# compatible model info
for model_id, model_info in iic_models_info.items():
_, model_name = model_id_to_group_owner_name(model_id)
models_info['damo/%s' % model_name] = models_info
# model_info key: model_id, value: model info such as framework task etc.
affected_pipeline_cases = []
affected_trainer_cases = []
@@ -255,8 +263,10 @@ def get_test_suites_to_run():
# ["PREPROCESSORS", "cv", "object_detection_scrfd", "SCRFDPreprocessor"]
# ["PREPROCESSORS", domain, preprocessor_name, class_name]
for model_id, model_info in models_info.items():
if model_info['preprocessor_type'] is not None and model_info[
'preprocessor_type'] == affected_register_module[2]:
if ('preprocessor_type' in model_info
and model_info['preprocessor_type'] is not None
and model_info['preprocessor_type']
== affected_register_module[2]):
task = model_info['task']
if task in task_pipeline_test_suite_map:
affected_pipeline_cases.extend(