2025-05-28 09:59:03 +08:00
|
|
|
import os
|
|
|
|
|
import subprocess
|
|
|
|
|
import sys
|
2026-06-03 18:39:39 +08:00
|
|
|
import tempfile
|
2025-05-28 09:59:03 +08:00
|
|
|
import unittest
|
2026-06-03 18:39:39 +08:00
|
|
|
from argparse import Namespace
|
|
|
|
|
from unittest import mock
|
2025-05-28 09:59:03 +08:00
|
|
|
|
2026-06-03 18:39:39 +08:00
|
|
|
from modelscope.hub.constants import TEMPORARY_FOLDER_NAME
|
2025-05-28 09:59:03 +08:00
|
|
|
from modelscope.utils.file_utils import get_modelscope_cache_dir
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class TestScanCacheCommand(unittest.TestCase):
|
|
|
|
|
"""Test cases for scancache command in ModelScope CLI."""
|
|
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
|
def setUpClass(cls):
|
|
|
|
|
"""Set up for tests."""
|
|
|
|
|
# download one file to ensure the cache directory exists
|
|
|
|
|
model_id = 'Qwen/Qwen3-0.6B'
|
|
|
|
|
cmd = f'python -m modelscope.cli.cli download --model {model_id} README.md'
|
|
|
|
|
subprocess.getstatusoutput(cmd)
|
|
|
|
|
|
|
|
|
|
def test_scan_default_dir(self):
|
|
|
|
|
cmd = 'python -m modelscope.cli.cli scan-cache'
|
|
|
|
|
stat, output = subprocess.getstatusoutput(cmd)
|
|
|
|
|
self.assertEqual(stat, 0)
|
[Feat & Refactor] Refactor hub and CLI modules (#1732)
* refactor(hub): shim layer delegating to modelscope-hub
- Replace hub/api.py (4674→250 lines) with shim inheriting LegacyHubApi
- Replace hub/snapshot_download.py, callback.py with thin shims
- Partial shim hub/file_download.py (retain http_get_file)
- Shim hub/constants.py and errors.py with legacy aliases
- Shim hub/git.py, repository.py, cache_manager.py, upload_*.py
- Migrate CLI entry to modelscope_hub.cli.main:run_cmd
- Adapt 6 CLI commands as modelscope_hub.cli_plugins
- Delete redundant CLI files (download/upload/login/create/etc)
- Add modelscope-hub>=0.2.0 dependency, Python>=3.10
- Add __getattr__ proxy for forward-compatible method access
- Propagate timeout/max_retries to internal LegacyClient
- Bridge MODELSCOPE_CREDENTIALS_PATH env var to HubConfig
* fix lint: isort/yapf formatting + exclude hub/api.py from hooks
* set modelscope-hub>=0.0.5
* remove unused code
* refactor(hub): standardize token naming — git_token vs token
Disambiguate git token and SDK/API token naming across the hub layer:
- ModelScopeConfig: get_token/save_token → get_git_token/save_git_token
(old names kept as deprecated aliases with DeprecationWarning)
- GitCommandWrapper: rename token params to git_token in clone/push/config
- Repository/DatasetRepository: auth_token → git_token (deprecated compat kept)
- data_loader.py: update caller to use get_git_token()
SDK token references (HubApi(token=...), get_cookies(access_token=...),
commit_scheduler.token) remain unchanged as they correctly use `token` naming.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
* remove(msdatasets): remove all Virgo-related implementation
Remove the entire Virgo dataset subsystem which is no longer needed:
- Remove VirgoDataset class and VirgoDownloader
- Remove VirgoAuthConfig and VirgoDatasetConfig
- Remove Hubs.virgo enum value
- Remove fetch_virgo_meta from DataMetaManager
- Remove download_virgo_files from DatasetContextConfig
- Remove test_virgo_dataset.py test file
- Clean up unused imports (pandas, MaxComputeUtil, valid_url, etc.)
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
* feat(hub): add OSS dataset operations and meta-file download to HubApi
Add methods that msdatasets depends on but don't belong in modelscope_hub:
- _legacy_request: internal helper combining legacy HTTP transport with
application-level envelope validation (Code/Data/Message)
- list_oss_dataset_objects: list OSS storage objects for a dataset
- delete_oss_dataset_object / delete_oss_dataset_dir: delete OSS objects
- fetch_meta_files_from_url: download and cache meta CSV/JSONL files
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
* fix imports issue
* fix: address PR review feedback
- cli/plugins.py: change --yes and --all flags to action='store_true'
- hub/git.py: replace os.linesep with .splitlines() for cross-platform safety
- hub/__init__.py: use is_file() with fallback for robust credentials path detection
* fix lint
* update ms hub version
* fix(ci): add PyPI official as fallback index for pip
Aliyun mirror may lag behind PyPI for newly published packages,
causing dependency resolution failures (e.g. modelscope-hub>=0.0.6).
Add pypi.org/simple as extra-index-url so new versions are immediately
available while keeping the Aliyun mirror as the primary source.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
* fix UTs
* remove unused UTs
* fix ut
* update modelscope-hub installation for source code
* fix UT
* fix uts
* fix ut
---------
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-06-09 20:00:20 +08:00
|
|
|
self.assertIn('cache_dir', output)
|
2025-05-28 09:59:03 +08:00
|
|
|
|
|
|
|
|
def test_scan_given_dir(self):
|
|
|
|
|
cmd = f'python -m modelscope.cli.cli scan-cache --dir {get_modelscope_cache_dir()}'
|
|
|
|
|
stat, output = subprocess.getstatusoutput(cmd)
|
|
|
|
|
self.assertEqual(stat, 0)
|
[Feat & Refactor] Refactor hub and CLI modules (#1732)
* refactor(hub): shim layer delegating to modelscope-hub
- Replace hub/api.py (4674→250 lines) with shim inheriting LegacyHubApi
- Replace hub/snapshot_download.py, callback.py with thin shims
- Partial shim hub/file_download.py (retain http_get_file)
- Shim hub/constants.py and errors.py with legacy aliases
- Shim hub/git.py, repository.py, cache_manager.py, upload_*.py
- Migrate CLI entry to modelscope_hub.cli.main:run_cmd
- Adapt 6 CLI commands as modelscope_hub.cli_plugins
- Delete redundant CLI files (download/upload/login/create/etc)
- Add modelscope-hub>=0.2.0 dependency, Python>=3.10
- Add __getattr__ proxy for forward-compatible method access
- Propagate timeout/max_retries to internal LegacyClient
- Bridge MODELSCOPE_CREDENTIALS_PATH env var to HubConfig
* fix lint: isort/yapf formatting + exclude hub/api.py from hooks
* set modelscope-hub>=0.0.5
* remove unused code
* refactor(hub): standardize token naming — git_token vs token
Disambiguate git token and SDK/API token naming across the hub layer:
- ModelScopeConfig: get_token/save_token → get_git_token/save_git_token
(old names kept as deprecated aliases with DeprecationWarning)
- GitCommandWrapper: rename token params to git_token in clone/push/config
- Repository/DatasetRepository: auth_token → git_token (deprecated compat kept)
- data_loader.py: update caller to use get_git_token()
SDK token references (HubApi(token=...), get_cookies(access_token=...),
commit_scheduler.token) remain unchanged as they correctly use `token` naming.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
* remove(msdatasets): remove all Virgo-related implementation
Remove the entire Virgo dataset subsystem which is no longer needed:
- Remove VirgoDataset class and VirgoDownloader
- Remove VirgoAuthConfig and VirgoDatasetConfig
- Remove Hubs.virgo enum value
- Remove fetch_virgo_meta from DataMetaManager
- Remove download_virgo_files from DatasetContextConfig
- Remove test_virgo_dataset.py test file
- Clean up unused imports (pandas, MaxComputeUtil, valid_url, etc.)
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
* feat(hub): add OSS dataset operations and meta-file download to HubApi
Add methods that msdatasets depends on but don't belong in modelscope_hub:
- _legacy_request: internal helper combining legacy HTTP transport with
application-level envelope validation (Code/Data/Message)
- list_oss_dataset_objects: list OSS storage objects for a dataset
- delete_oss_dataset_object / delete_oss_dataset_dir: delete OSS objects
- fetch_meta_files_from_url: download and cache meta CSV/JSONL files
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
* fix imports issue
* fix: address PR review feedback
- cli/plugins.py: change --yes and --all flags to action='store_true'
- hub/git.py: replace os.linesep with .splitlines() for cross-platform safety
- hub/__init__.py: use is_file() with fallback for robust credentials path detection
* fix lint
* update ms hub version
* fix(ci): add PyPI official as fallback index for pip
Aliyun mirror may lag behind PyPI for newly published packages,
causing dependency resolution failures (e.g. modelscope-hub>=0.0.6).
Add pypi.org/simple as extra-index-url so new versions are immediately
available while keeping the Aliyun mirror as the primary source.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
* fix UTs
* remove unused UTs
* fix ut
* update modelscope-hub installation for source code
* fix UT
* fix uts
* fix ut
---------
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-06-09 20:00:20 +08:00
|
|
|
self.assertIn('cache_dir', output)
|
2025-05-28 09:59:03 +08:00
|
|
|
|
|
|
|
|
def test_scan_not_exist_dir(self):
|
|
|
|
|
cmd = 'python -m modelscope.cli.cli scan-cache --dir /fake/cache/path'
|
|
|
|
|
stat, output = subprocess.getstatusoutput(cmd)
|
|
|
|
|
self.assertEqual(stat, 0)
|
[Feat & Refactor] Refactor hub and CLI modules (#1732)
* refactor(hub): shim layer delegating to modelscope-hub
- Replace hub/api.py (4674→250 lines) with shim inheriting LegacyHubApi
- Replace hub/snapshot_download.py, callback.py with thin shims
- Partial shim hub/file_download.py (retain http_get_file)
- Shim hub/constants.py and errors.py with legacy aliases
- Shim hub/git.py, repository.py, cache_manager.py, upload_*.py
- Migrate CLI entry to modelscope_hub.cli.main:run_cmd
- Adapt 6 CLI commands as modelscope_hub.cli_plugins
- Delete redundant CLI files (download/upload/login/create/etc)
- Add modelscope-hub>=0.2.0 dependency, Python>=3.10
- Add __getattr__ proxy for forward-compatible method access
- Propagate timeout/max_retries to internal LegacyClient
- Bridge MODELSCOPE_CREDENTIALS_PATH env var to HubConfig
* fix lint: isort/yapf formatting + exclude hub/api.py from hooks
* set modelscope-hub>=0.0.5
* remove unused code
* refactor(hub): standardize token naming — git_token vs token
Disambiguate git token and SDK/API token naming across the hub layer:
- ModelScopeConfig: get_token/save_token → get_git_token/save_git_token
(old names kept as deprecated aliases with DeprecationWarning)
- GitCommandWrapper: rename token params to git_token in clone/push/config
- Repository/DatasetRepository: auth_token → git_token (deprecated compat kept)
- data_loader.py: update caller to use get_git_token()
SDK token references (HubApi(token=...), get_cookies(access_token=...),
commit_scheduler.token) remain unchanged as they correctly use `token` naming.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
* remove(msdatasets): remove all Virgo-related implementation
Remove the entire Virgo dataset subsystem which is no longer needed:
- Remove VirgoDataset class and VirgoDownloader
- Remove VirgoAuthConfig and VirgoDatasetConfig
- Remove Hubs.virgo enum value
- Remove fetch_virgo_meta from DataMetaManager
- Remove download_virgo_files from DatasetContextConfig
- Remove test_virgo_dataset.py test file
- Clean up unused imports (pandas, MaxComputeUtil, valid_url, etc.)
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
* feat(hub): add OSS dataset operations and meta-file download to HubApi
Add methods that msdatasets depends on but don't belong in modelscope_hub:
- _legacy_request: internal helper combining legacy HTTP transport with
application-level envelope validation (Code/Data/Message)
- list_oss_dataset_objects: list OSS storage objects for a dataset
- delete_oss_dataset_object / delete_oss_dataset_dir: delete OSS objects
- fetch_meta_files_from_url: download and cache meta CSV/JSONL files
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
* fix imports issue
* fix: address PR review feedback
- cli/plugins.py: change --yes and --all flags to action='store_true'
- hub/git.py: replace os.linesep with .splitlines() for cross-platform safety
- hub/__init__.py: use is_file() with fallback for robust credentials path detection
* fix lint
* update ms hub version
* fix(ci): add PyPI official as fallback index for pip
Aliyun mirror may lag behind PyPI for newly published packages,
causing dependency resolution failures (e.g. modelscope-hub>=0.0.6).
Add pypi.org/simple as extra-index-url so new versions are immediately
available while keeping the Aliyun mirror as the primary source.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
* fix UTs
* remove unused UTs
* fix ut
* update modelscope-hub installation for source code
* fix UT
* fix uts
* fix ut
---------
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-06-09 20:00:20 +08:00
|
|
|
self.assertIn('0 repo(s)', output)
|
2025-05-28 09:59:03 +08:00
|
|
|
|
|
|
|
|
|
2026-06-03 18:39:39 +08:00
|
|
|
class TestClearCacheCommand(unittest.TestCase):
|
|
|
|
|
"""clear-cache must resolve the right cache root in both scenarios:
|
|
|
|
|
* MODELSCOPE_CACHE set -> $MODELSCOPE_CACHE/{models,datasets}/<id>
|
|
|
|
|
* MODELSCOPE_CACHE unset -> ~/.cache/modelscope/hub/{models,datasets}/<id>
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
def _clear_and_assert(self, root, kind, entity_id):
|
|
|
|
|
from modelscope.cli.clearcache import ClearCacheCMD
|
|
|
|
|
sub = {'model': 'models', 'dataset': 'datasets'}[kind]
|
|
|
|
|
entity_dir = os.path.join(root, sub, entity_id)
|
|
|
|
|
temp_dir = os.path.join(root, sub, TEMPORARY_FOLDER_NAME, entity_id)
|
|
|
|
|
os.makedirs(entity_dir)
|
|
|
|
|
os.makedirs(temp_dir)
|
|
|
|
|
args = Namespace(
|
|
|
|
|
model=entity_id if kind == 'model' else None,
|
|
|
|
|
dataset=entity_id if kind == 'dataset' else None)
|
|
|
|
|
with mock.patch('builtins.input', return_value='Y'):
|
|
|
|
|
ClearCacheCMD(args).execute()
|
|
|
|
|
self.assertFalse(os.path.exists(entity_dir))
|
|
|
|
|
self.assertFalse(os.path.exists(temp_dir))
|
|
|
|
|
|
|
|
|
|
def test_env_cache(self):
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmp, \
|
|
|
|
|
mock.patch.dict(os.environ,
|
|
|
|
|
{'MODELSCOPE_CACHE': tmp}, clear=False):
|
|
|
|
|
for kind in ('model', 'dataset'):
|
|
|
|
|
with self.subTest(kind=kind):
|
|
|
|
|
self._clear_and_assert(tmp, kind, f'org/{kind}')
|
|
|
|
|
|
|
|
|
|
def test_default_cache(self):
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
|
env = {
|
|
|
|
|
k: v
|
|
|
|
|
for k, v in os.environ.items() if k != 'MODELSCOPE_CACHE'
|
|
|
|
|
}
|
|
|
|
|
env['HOME'] = tmp
|
|
|
|
|
with mock.patch.dict(os.environ, env, clear=True):
|
|
|
|
|
self._clear_and_assert(
|
|
|
|
|
os.path.join(tmp, '.cache', 'modelscope', 'hub'), 'model',
|
|
|
|
|
'org/repo')
|
|
|
|
|
|
|
|
|
|
|
2025-05-28 09:59:03 +08:00
|
|
|
if __name__ == '__main__':
|
|
|
|
|
unittest.main()
|