Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2026-08-29 10:08:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

add scan cache unit test

Browse Source
This commit is contained in:
Yunnglin
2025-05-27 14:23:30 +08:00
parent 15931ed276
commit 9e666d1c93
3 changed files with 91 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
modelscope/hub/cache_manager.py
View File

@@ -10,7 +10,7 @@ from modelscope.hub.utils.caching import ModelFileSystemCache
from modelscope.hub.utils.utils import (convert_readable_size,
format_timesince, tabulate)
from modelscope.utils.constant import REPO_TYPE_DATASET, REPO_TYPE_MODEL
from modelscope.utils.file_utils import get_default_modelscope_cache_dir
from modelscope.utils.file_utils import get_modelscope_cache_dir
from modelscope.utils.logger import get_logger
logger = get_logger()
@@ -42,6 +42,7 @@ class CachedFileInfo:
file_name: str
file_path: Path
file_revision_hash: str
blob_path: Path
size_on_disk: int
@@ -238,11 +239,11 @@ class ModelScopeCacheInfo:
ModelScopeCacheInfo(...)
>>> print(ms_cache_info.export_as_table())
REPO ID REPO TYPE REVISION SIZE ON DISK NB FILES LAST_MODIFIED LOCAL PATH
--------------------------------------------------- --------- ---------- ------------ -------- ------------- -------------------------------------------------------------
damo/bert-base-chinese model master 2.7M 5 1 week ago ~/.cache/modelscope/hub/models--damo--bert-base-chinese/...
damo/structured-bert model master 8.8K 1 1 week ago ~/.cache/modelscope/hub/models--damo--structured-bert/...
damo/t5-base model master 893.8M 4 7 months ago ~/.cache/modelscope/hub/models--damo--t5-base/...
REPO ID REPO TYPE REVISION SIZE ON DISK NB FILES LAST_MODIFIED LOCAL PATH
---------------------- --------- ---------- ------------ -------- ------------- -------------------------------------------------------------
damo/bert-base-chinese model master 2.7M 5 1 week ago ~/.cache/modelscope/hub/models--damo--bert-base-chinese/...
damo/structured-bert model master 8.8K 1 1 week ago ~/.cache/modelscope/hub/models--damo--structured-bert/...
damo/t5-base model master 893.8M 4 7 months ago ~/.cache/modelscope/hub/models--damo--t5-base/...
```
Returns:
@@ -351,7 +352,7 @@ def scan_cache_dir(
Returns: a [`ModelScopeCacheInfo`] object.
"""
if cache_dir is None:
cache_dir = get_default_modelscope_cache_dir()
cache_dir = get_modelscope_cache_dir()
cache_dir = Path(cache_dir).expanduser().resolve()
if not cache_dir.exists():
@@ -391,7 +392,7 @@ def scan_cache_dir(
# Also check for repos directly in cache_dir (older structure)
# If the repo is not in models/ or datasets/, assume it's a model repo
other_repos, other_warnings = _scan_dir(
cache_dir, repo_type=REPO_TYPE_MODEL, extended=True)
cache_dir, repo_type=REPO_TYPE_MODEL, inplace=True)
repos.update(other_repos)
warnings.extend(other_warnings)
@@ -415,13 +416,13 @@ def _is_valid_dir(dir: Path) -> bool:
return True
def _scan_dir(dir: Path, repo_type: str, extended: bool = False):
def _scan_dir(dir: Path, repo_type: str, inplace: bool = False):
"""Scan a directory for cached repos and return a set of [`~CachedRepoInfo`] and warnings."""
repos = set()
warnings = []
for owner_dir in dir.iterdir():
# not extend scan the following dirs
if extended and owner_dir.name in ['models', 'datasets', 'hub']:
# not extend scan the following dirs when scan current dir
if inplace and owner_dir.name in ['models', 'datasets', 'hub']:
continue
if not _is_valid_dir(owner_dir):
continue
@@ -467,6 +468,7 @@ def _scan_cached_repo(repo_path: Path,
# Process all cached files
for cached_file in cached_files:
file_path = os.path.join(repo_path, cached_file['Path'])
file_revision_hash = cached_file.get('Revision', '')
if not os.path.exists(file_path):
continue
@@ -478,6 +480,7 @@ def _scan_cached_repo(repo_path: Path,
CachedFileInfo(
file_name=os.path.basename(cached_file['Path']),
file_path=blob_path,
file_revision_hash=file_revision_hash,
size_on_disk=blob_stats[blob_path].st_size,
blob_path=blob_path,
blob_last_accessed=blob_stats[blob_path].st_atime,

36
tests/cli/test_scancache_cmd.py Normal file
View File

@@ -0,0 +1,36 @@
import os
import subprocess
import sys
import unittest
from modelscope.utils.file_utils import get_modelscope_cache_dir
class TestScanCacheCommand(unittest.TestCase):
"""Test cases for scancache command in ModelScope CLI."""
def setUp(self):
"""Set up for tests."""
self.fake_cache_dir = '/fake/cache/path'
def test_scan_default_dir(self):
cmd = 'python -m modelscope.cli.cli scan-cache'
stat, output = subprocess.getstatusoutput(cmd)
self.assertEqual(stat, 0)
self.assertIn('Done', output)
def test_scan_given_dir(self):
cmd = f'python -m modelscope.cli.cli scan-cache --dir {get_modelscope_cache_dir()}'
stat, output = subprocess.getstatusoutput(cmd)
self.assertEqual(stat, 0)
self.assertIn('Done', output)
def test_scan_not_exist_dir(self):
cmd = f'python -m modelscope.cli.cli scan-cache --dir {self.fake_cache_dir}'
stat, output = subprocess.getstatusoutput(cmd)
self.assertEqual(stat, 0)
self.assertIn('not found', output)
if __name__ == '__main__':
unittest.main()

41
tests/hub/test_hub_scan_cache.py Normal file
View File

@@ -0,0 +1,41 @@
# Copyright (c) Alibaba, Inc. and its affiliates.
import unittest
from modelscope.hub.cache_manager import scan_cache_dir
from modelscope.hub.errors import CacheNotFound
from modelscope.utils.file_utils import get_modelscope_cache_dir
from modelscope.utils.logger import get_logger
logger = get_logger()
class HubScanCacheTest(unittest.TestCase):
def setUp(self):
"""Set up for tests."""
self.fake_cache_dir = '/fake/cache/path'
def test_scan_default_dir(self):
"""Test scanning the default cache directory."""
try:
res_info = scan_cache_dir()
logger.info(res_info.export_as_table())
except Exception as e:
self.fail(f'Scanning default cache directory failed: {e}')
def test_scan_given_dir(self):
"""Test scanning a given cache directory."""
try:
scan_cache_dir(get_modelscope_cache_dir())
logger.info('Done')
except Exception as e:
self.fail(f'Scanning given cache directory failed: {e}')
def test_scan_not_exist_dir(self):
"""Test scanning a non-existent cache directory."""
with self.assertRaises(CacheNotFound):
scan_cache_dir('/non/existent/path')
if __name__ == '__main__':
unittest.main()