diff --git a/modelscope/cli/clearcache.py b/modelscope/cli/clearcache.py index 7b89103b..dcd3d1df 100644 --- a/modelscope/cli/clearcache.py +++ b/modelscope/cli/clearcache.py @@ -6,6 +6,7 @@ from pathlib import Path from modelscope.cli.base import CLICommand from modelscope.hub.constants import TEMPORARY_FOLDER_NAME +from modelscope.hub.utils.utils import get_model_masked_directory def subparser_func(args): @@ -99,8 +100,13 @@ class ClearCacheCMD(CLICommand): def _remove_directory(self, path): if os.path.exists(path): try: - shutil.rmtree(path) - print(f'Cache folder {path} removed.') + if os.path.islink(path): + shutil.rmtree(os.readlink(path)) + os.remove(path) + print(f'Cache and link for {path} removed.') + else: + shutil.rmtree(path) + print(f'Cache folder {path} removed.') return True except Exception as e: print(f'An error occurred while clearing cache at {path}: {e}') diff --git a/modelscope/hub/snapshot_download.py b/modelscope/hub/snapshot_download.py index 1fe94b82..dbec55c1 100644 --- a/modelscope/hub/snapshot_download.py +++ b/modelscope/hub/snapshot_download.py @@ -11,7 +11,8 @@ from typing import Dict, List, Optional, Union from modelscope.hub.api import HubApi, ModelScopeConfig from modelscope.hub.errors import InvalidParameter from modelscope.hub.utils.caching import ModelFileSystemCache -from modelscope.hub.utils.utils import model_id_to_group_owner_name +from modelscope.hub.utils.utils import (get_model_masked_directory, + model_id_to_group_owner_name) from modelscope.utils.constant import (DEFAULT_DATASET_REVISION, DEFAULT_MODEL_REVISION, REPO_TYPE_DATASET, REPO_TYPE_MODEL, @@ -219,9 +220,9 @@ def _snapshot_download( if cookies is None: cookies = ModelScopeConfig.get_cookies() repo_files = [] - directory = os.path.join(system_cache, 'hub', repo_id) - print(f'Downloading Model to directory: {directory}') if repo_type == REPO_TYPE_MODEL: + directory = os.path.join(system_cache, 'hub', repo_id) + print(f'Downloading Model to directory: {directory}') revision_detail = _api.get_valid_revision_detail( repo_id, revision=revision, cookies=cookies) revision = revision_detail['Revision'] @@ -259,6 +260,18 @@ def _snapshot_download( allow_file_pattern=allow_file_pattern, ignore_patterns=ignore_patterns, allow_patterns=allow_patterns) + if '.' in repo_id: + masked_directory = get_model_masked_directory( + directory, repo_id) + logger.info( + f'Creating symbolic link {masked_directory} -> {directory}.' + ) + try: + os.symlink(os.path.abspath(masked_directory), directory) + except OSError as e: + logger.warning( + f'Failed to create symbolic link {masked_directory} -> {directory}: {e}' + ) elif repo_type == REPO_TYPE_DATASET: directory = os.path.join(system_cache, 'datasets', repo_id) diff --git a/modelscope/hub/utils/utils.py b/modelscope/hub/utils/utils.py index 3c3c75da..6cc53285 100644 --- a/modelscope/hub/utils/utils.py +++ b/modelscope/hub/utils/utils.py @@ -29,6 +29,15 @@ def model_id_to_group_owner_name(model_id): return group_or_owner, name +# during model download, the '.' would be converted to '___' to produce +# actual physical (masked) directory for storage +def get_model_masked_directory(directory, model_id): + parts = directory.rsplit('/', 2) + # this is the actual directory the model files are located. + masked_directory = os.path.join(parts[0], model_id.replace('.', '___')) + return masked_directory + + def get_cache_dir(model_id: Optional[str] = None): """cache dir precedence: function parameter > environment > ~/.cache/modelscope/hub