From 931d0dac643191d09f827393dacb03b3a8e88417 Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Thu, 13 Mar 2025 11:34:38 +0800 Subject: [PATCH 1/3] add download count for ci-test --- modelscope/hub/snapshot_download.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/modelscope/hub/snapshot_download.py b/modelscope/hub/snapshot_download.py index 75bcb991..8923e9e3 100644 --- a/modelscope/hub/snapshot_download.py +++ b/modelscope/hub/snapshot_download.py @@ -238,10 +238,8 @@ def _snapshot_download( headers = { 'user-agent': ModelScopeConfig.get_user_agent(user_agent=user_agent, ), + 'snapshot-identifier': str(uuid.uuid4()), } - if 'CI_TEST' not in os.environ: - # To count the download statistics, to add the snapshot-identifier as a header. - headers['snapshot-identifier'] = str(uuid.uuid4()) _api = HubApi() if cookies is None: cookies = ModelScopeConfig.get_cookies() @@ -258,12 +256,12 @@ def _snapshot_download( repo_id, revision=revision, cookies=cookies) revision = revision_detail['Revision'] - snapshot_header = headers if 'CI_TEST' in os.environ else { - **headers, - **{ - 'Snapshot': 'True' - } - } + # Add snapshot-ci-test for counting the ci test download + if 'CI_TEST' in os.environ: + snapshot_header = {**headers, **{'snapshot-ci-test': 'True'}} + else: + snapshot_header = {**headers, **{'Snapshot': 'True'}} + if cache.cached_model_revision is not None: snapshot_header[ 'cached_model_revision'] = cache.cached_model_revision From d4d4c3aa0633b22e8befbd496e0ee426c92d727c Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Thu, 13 Mar 2025 20:10:36 +0800 Subject: [PATCH 2/3] update get_endpoint() to avoid modelscope_domain is empty string --- modelscope/hub/utils/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modelscope/hub/utils/utils.py b/modelscope/hub/utils/utils.py index 7d377013..0fd078b0 100644 --- a/modelscope/hub/utils/utils.py +++ b/modelscope/hub/utils/utils.py @@ -106,8 +106,9 @@ def get_release_datetime(): def get_endpoint(): - modelscope_domain = os.getenv('MODELSCOPE_DOMAIN', - DEFAULT_MODELSCOPE_DOMAIN) + modelscope_domain = os.getenv( + 'MODELSCOPE_DOMAIN', + DEFAULT_MODELSCOPE_DOMAIN) or DEFAULT_MODELSCOPE_DOMAIN return MODELSCOPE_URL_SCHEME + modelscope_domain From e8150fbac908c091ab547c9fbd65bfc8d5b5388c Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Thu, 13 Mar 2025 20:13:23 +0800 Subject: [PATCH 3/3] add png for DATASET_LFS_SUFFIX --- modelscope/utils/repo_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modelscope/utils/repo_utils.py b/modelscope/utils/repo_utils.py index 55d16251..d14adbf8 100644 --- a/modelscope/utils/repo_utils.py +++ b/modelscope/utils/repo_utils.py @@ -49,6 +49,7 @@ DATASET_LFS_SUFFIX = [ '.jack', '.jpeg', '.jpg', + '.png', '.jsonl', '.joblib', '.lz4',