diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index e11f2de5..ac66e11c 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -429,6 +429,30 @@ class HubApi: use_cookies: Union[bool, CookieJar] = False) -> List[str]: """Get model branch and tags. + Args: + model_id (str): The model id + cutoff_timestamp (int): Tags created before the cutoff will be included. + The timestamp is represented by the seconds elapsed from the epoch time. + use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True, + will load cookie from local. Defaults to False. + + Returns: + Tuple[List[str], List[str]]: Return list of branch name and tags + """ + tags_details = self.list_model_revisions_detail(model_id=model_id, + cutoff_timestamp=cutoff_timestamp, + use_cookies=use_cookies) + tags = [x['Revision'] for x in tags_details + ] if tags_details else [] + return tags + + def list_model_revisions_detail( + self, + model_id: str, + cutoff_timestamp: Optional[int] = None, + use_cookies: Union[bool, CookieJar] = False) -> List[str]: + """Get model branch and tags. + Args: model_id (str): The model id cutoff_timestamp (int): Tags created before the cutoff will be included. @@ -450,66 +474,84 @@ class HubApi: raise_on_error(d) info = d[API_RESPONSE_FIELD_DATA] # tags returned from backend are guaranteed to be ordered by create-time - tags = [x['Revision'] for x in info['RevisionMap']['Tags'] - ] if info['RevisionMap']['Tags'] else [] - return tags + return info['RevisionMap']['Tags'] - def get_valid_revision(self, - model_id: str, - revision=None, - cookies: Optional[CookieJar] = None): + def get_branch_tag_detail(self, details, name): + for item in details: + if item['Revision'] == name: + return item + return None + + def get_valid_revision_detail(self, + model_id: str, + revision=None, + cookies: Optional[CookieJar] = None): release_timestamp = get_release_datetime() current_timestamp = int(round(datetime.datetime.now().timestamp())) # for active development in library codes (non-release-branches), release_timestamp # is set to be a far-away-time-in-the-future, to ensure that we shall # get the master-HEAD version from model repo by default (when no revision is provided) + all_branches_detail, all_tags_detail = self.get_model_branches_and_tags_details( + model_id, use_cookies=False if cookies is None else cookies) + all_branches = [x['Revision'] for x in all_branches_detail] if all_branches_detail else [] + all_tags = [x['Revision'] for x in all_tags_detail] if all_tags_detail else [] if release_timestamp > current_timestamp + ONE_YEAR_SECONDS: - branches, tags = self.get_model_branches_and_tags( - model_id, use_cookies=False if cookies is None else cookies) if revision is None: revision = MASTER_MODEL_BRANCH logger.info( 'Model revision not specified, use default: %s in development mode' % revision) - if revision not in branches and revision not in tags: + if revision not in all_branches and revision not in all_tags: raise NotExistError('The model: %s has no revision : %s .' % (model_id, revision)) + + revision_detail = self.get_branch_tag_detail(all_tags_detail, revision) + if revision_detail is None: + revision_detail = self.get_branch_tag_detail(all_branches_detail, revision) logger.info('Development mode use revision: %s' % revision) else: - all_revisions = self.list_model_revisions( - model_id, - cutoff_timestamp=current_timestamp, - use_cookies=False if cookies is None else cookies) - if len(all_revisions) == 0: + if len(all_tags_detail) == 0: # use no revision use master as default. if revision is None or revision == MASTER_MODEL_BRANCH: revision = MASTER_MODEL_BRANCH else: raise NotExistError('The model: %s has no revision: %s !' % (model_id, revision)) + revision_detail = self.get_branch_tag_detail(all_branches_detail, revision) else: if revision is None: # user not specified revision, use latest revision before release time - revisions = self.list_model_revisions( - model_id, - cutoff_timestamp=release_timestamp, - use_cookies=False if cookies is None else cookies) - if len(revisions) > 0: - revision = revisions[0] # use latest revision before release time. + revisions_detail = [x for x in + all_tags_detail if x['CreatedAt'] <= release_timestamp] if all_tags_detail else [] # noqa E501 + if len(revisions_detail) > 0: + revision = revisions_detail[0]['Revision'] # use latest revision before release time. + revision_detail = revisions_detail[0] else: revision = MASTER_MODEL_BRANCH - vl = '[%s]' % ','.join(all_revisions) + revision_detail = self.get_branch_tag_detail(all_branches_detail, revision) + vl = '[%s]' % ','.join(all_tags) logger.warning('Model revision should be specified from revisions: %s' % (vl)) logger.warning('Model revision not specified, use revision: %s' % revision) else: # use user-specified revision - if revision not in all_revisions: + if revision not in all_tags: if revision == MASTER_MODEL_BRANCH: logger.warning('Using the master branch is fragile, please use it with caution!') + revision_detail = self.get_branch_tag_detail(all_branches_detail, revision) else: - vl = '[%s]' % ','.join(all_revisions) + vl = '[%s]' % ','.join(all_tags) raise NotExistError('The model: %s has no revision: %s valid are: %s!' % (model_id, revision, vl)) + else: + revision_detail = self.get_branch_tag_detail(all_tags_detail, revision) logger.info('Use user-specified model revision: %s' % revision) - return revision + return revision_detail - def get_model_branches_and_tags( + def get_valid_revision(self, + model_id: str, + revision=None, + cookies: Optional[CookieJar] = None): + return self.get_valid_revision_detail(model_id=model_id, + revision=revision, + cookies=cookies)['Revision'] + + def get_model_branches_and_tags_details( self, model_id: str, use_cookies: Union[bool, CookieJar] = False, @@ -533,10 +575,29 @@ class HubApi: d = r.json() raise_on_error(d) info = d[API_RESPONSE_FIELD_DATA] - branches = [x['Revision'] for x in info['RevisionMap']['Branches'] - ] if info['RevisionMap']['Branches'] else [] - tags = [x['Revision'] for x in info['RevisionMap']['Tags'] - ] if info['RevisionMap']['Tags'] else [] + return info['RevisionMap']['Branches'], info['RevisionMap']['Tags'] + + def get_model_branches_and_tags( + self, + model_id: str, + use_cookies: Union[bool, CookieJar] = False, + ) -> Tuple[List[str], List[str]]: + """Get model branch and tags. + + Args: + model_id (str): The model id + use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True, + will load cookie from local. Defaults to False. + + Returns: + Tuple[List[str], List[str]]: Return list of branch name and tags + """ + branches_detail, tags_detail = self.get_model_branches_and_tags_details(model_id=model_id, + use_cookies=use_cookies) + branches = [x['Revision'] for x in branches_detail + ] if branches_detail else [] + tags = [x['Revision'] for x in tags_detail + ] if tags_detail else [] return branches, tags def get_model_files(self, diff --git a/modelscope/hub/snapshot_download.py b/modelscope/hub/snapshot_download.py index aafd4cd9..dd332c6b 100644 --- a/modelscope/hub/snapshot_download.py +++ b/modelscope/hub/snapshot_download.py @@ -94,8 +94,9 @@ def snapshot_download(model_id: str, _api = HubApi() if cookies is None: cookies = ModelScopeConfig.get_cookies() - revision = _api.get_valid_revision( + revision_detail = _api.get_valid_revision_detail( model_id, revision=revision, cookies=cookies) + revision = revision_detail['Revision'] snapshot_header = headers if 'CI_TEST' in os.environ else { **headers, @@ -165,6 +166,6 @@ def snapshot_download(model_id: str, # put file into to cache cache.put_file(model_file, temp_file) - cache.save_model_version(revision=revision) + cache.save_model_version(revision_info=revision_detail) return os.path.join(cache.get_root_location()) diff --git a/modelscope/hub/utils/caching.py b/modelscope/hub/utils/caching.py index 78f3929d..cfa20f07 100644 --- a/modelscope/hub/utils/caching.py +++ b/modelscope/hub/utils/caching.py @@ -5,6 +5,7 @@ import os import pickle import tempfile from shutil import move, rmtree +from typing import Dict from modelscope.utils.logger import get_logger @@ -159,11 +160,13 @@ class ModelFileSystemCache(FileSystemCache): else: return None - def save_model_version(self, revision: str): + def save_model_version(self, revision_info: Dict): model_version_file_path = os.path.join( self.cache_root_location, FileSystemCache.MODEL_VERSION_FILE_NAME) with open(model_version_file_path, 'w') as f: - f.write(revision) + version_info_str = 'Revision:%s,CreatedAt:%s' % ( + revision_info['Revision'], revision_info['CreatedAt']) + f.write(version_info_str) def get_model_id(self): return self.model_meta[FileSystemCache.MODEL_META_MODEL_ID] diff --git a/tests/run_analysis.py b/tests/run_analysis.py index ac0f2ac9..76a665ff 100644 --- a/tests/run_analysis.py +++ b/tests/run_analysis.py @@ -2,7 +2,6 @@ import os import subprocess -import sys from fnmatch import fnmatch from trainers.model_trainer_map import model_trainer_map @@ -12,9 +11,9 @@ from utils.source_file_analyzer import (get_all_register_modules, get_import_map) from modelscope.hub.api import HubApi -from modelscope.hub.errors import NotExistError from modelscope.hub.file_download import model_file_download -from modelscope.hub.utils.utils import get_cache_dir +from modelscope.hub.utils.utils import (get_cache_dir, + model_id_to_group_owner_name) from modelscope.utils.config import Config from modelscope.utils.constant import ModelFile from modelscope.utils.logger import get_logger @@ -27,10 +26,14 @@ def get_models_info(groups: list) -> dict: api = HubApi() for group in groups: page = 1 + total_count = 0 while True: query_result = api.list_models(group, page, 100) - models.extend(query_result['Models']) - if len(models) >= query_result['TotalCount']: + if query_result['Models'] is not None: + models.extend(query_result['Models']) + elif total_count != 0: + total_count = query_result['TotalCount'] + if len(models) >= total_count: break page += 1 cache_root = get_cache_dir() @@ -218,7 +221,12 @@ def get_test_suites_to_run(): all_register_modules) # task_pipeline_test_suite_map key: pipeline task, value: case file path # trainer_test_suite_map key: trainer_name, value: case file path - models_info = get_models_info(['damo']) + iic_models_info = get_models_info(['iic']) + models_info = {} + # compatible model info + for model_id, model_info in iic_models_info.items(): + _, model_name = model_id_to_group_owner_name(model_id) + models_info['damo/%s' % model_name] = models_info # model_info key: model_id, value: model info such as framework task etc. affected_pipeline_cases = [] affected_trainer_cases = [] @@ -255,8 +263,10 @@ def get_test_suites_to_run(): # ["PREPROCESSORS", "cv", "object_detection_scrfd", "SCRFDPreprocessor"] # ["PREPROCESSORS", domain, preprocessor_name, class_name] for model_id, model_info in models_info.items(): - if model_info['preprocessor_type'] is not None and model_info[ - 'preprocessor_type'] == affected_register_module[2]: + if ('preprocessor_type' in model_info + and model_info['preprocessor_type'] is not None + and model_info['preprocessor_type'] + == affected_register_module[2]): task = model_info['task'] if task in task_pipeline_test_suite_map: affected_pipeline_cases.extend(