Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2026-09-01 19:49:03 +02:00
Code Issues Packages Projects Releases Wiki Activity

Use legacy cache (#1215)

Browse Source 
(cherry picked from commit b5bb6d7bb0)
This commit is contained in:
tastelikefeet
2025-02-07 17:31:32 +08:00
committed by yuze.zyz
parent 2df80c0c40
commit 8282a236a0
4 changed files with 81 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docker/Dockerfile.ubuntu
View File

@@ -67,5 +67,5 @@ RUN sh /tmp/install.sh {version_args} && \
ENV SETUPTOOLS_USE_DISTUTILS=stdlib
ENV VLLM_USE_MODELSCOPE=True
ENV LMDEPLOY_USE_MODELSCOPE=True
ENV MODELSCOPE_CACHE=/mnt/workspace/.cache/modelscope
ENV MODELSCOPE_CACHE=/mnt/workspace/.cache/modelscope/hub
SHELL ["/bin/bash", "-c"]

41
modelscope/hub/file_download.py
View File

@@ -4,6 +4,7 @@ import copy
import hashlib
import io
import os
import shutil
import tempfile
import urllib
import uuid
@@ -286,6 +287,41 @@ def _repo_file_download(
temporary_cache_dir, cache, headers, cookies)
def move_legacy_cache_to_standard_dir(cache_dir: str, model_id: str):
if cache_dir.endswith(os.path.sep):
cache_dir = cache_dir.strip(os.path.sep)
legacy_cache_root = os.path.dirname(cache_dir)
base_name = os.path.basename(cache_dir)
if base_name == 'datasets':
# datasets will not be not affected
return
if not legacy_cache_root.endswith('hub'):
# Two scenarios:
# We have restructured ModelScope cache directory,
# Scenery 1:
# When MODELSCOPE_CACHE is not set, the default directory remains
# the same at ~/.cache/modelscope/hub
# Scenery 2:
# When MODELSCOPE_CACHE is not set, the cache directory is moved from
# $MODELSCOPE_CACHE/hub to $MODELSCOPE_CACHE/. In this case,
# we will be migrating the hub directory accordingly.
legacy_cache_root = os.path.join(legacy_cache_root, 'hub')
group_or_owner, name = model_id_to_group_owner_name(model_id)
name = name.replace('.', '___')
temporary_cache_dir = os.path.join(cache_dir, group_or_owner, name)
legacy_cache_dir = os.path.join(legacy_cache_root, group_or_owner, name)
if os.path.exists(
legacy_cache_dir) and not os.path.exists(temporary_cache_dir):
logger.info(
f'Legacy cache dir exists: {legacy_cache_dir}, move to {temporary_cache_dir}'
)
try:
shutil.move(legacy_cache_dir, temporary_cache_dir)
except Exception: # noqa
# Failed, skip
pass
def create_temporary_directory_and_cache(model_id: str,
local_dir: str = None,
cache_dir: str = None,
@@ -294,6 +330,10 @@ def create_temporary_directory_and_cache(model_id: str,
default_cache_root = get_model_cache_root()
elif repo_type == REPO_TYPE_DATASET:
default_cache_root = get_dataset_cache_root()
else:
raise ValueError(
f'repo_type only support model and dataset, but now is : {repo_type}'
)
group_or_owner, name = model_id_to_group_owner_name(model_id)
if local_dir is not None:
@@ -302,6 +342,7 @@ def create_temporary_directory_and_cache(model_id: str,
else:
if cache_dir is None:
cache_dir = default_cache_root
move_legacy_cache_to_standard_dir(cache_dir, model_id)
if isinstance(cache_dir, Path):
cache_dir = str(cache_dir)
temporary_cache_dir = os.path.join(cache_dir, TEMPORARY_FOLDER_NAME,

3
modelscope/hub/snapshot_download.py
View File

@@ -17,7 +17,6 @@ from modelscope.hub.utils.utils import (get_model_masked_directory,
model_id_to_group_owner_name)
from modelscope.utils.constant import (DEFAULT_DATASET_REVISION,
DEFAULT_MODEL_REVISION,
DEFAULT_REPOSITORY_REVISION,
REPO_TYPE_DATASET, REPO_TYPE_MODEL,
REPO_TYPE_SUPPORT)
from modelscope.utils.file_utils import get_modelscope_cache_dir
@@ -246,7 +245,6 @@ def _snapshot_download(
_api = HubApi()
if cookies is None:
cookies = ModelScopeConfig.get_cookies()
repo_files = []
if repo_type == REPO_TYPE_MODEL:
directory = os.path.abspath(
local_dir) if local_dir is not None else os.path.join(
@@ -313,7 +311,6 @@ def _snapshot_download(
local_dir) if local_dir else os.path.join(
system_cache, 'datasets', *repo_id.split('/'))
print(f'Downloading Dataset to directory: {directory}')
group_or_owner, name = model_id_to_group_owner_name(repo_id)
revision_detail = revision or DEFAULT_DATASET_REVISION

59
modelscope/utils/hf_util/patcher.py
View File

@@ -47,29 +47,48 @@ def get_all_imported_modules():
pass
if importlib.util.find_spec('peft') is not None:
import peft
attributes = dir(peft)
imports = [attr for attr in attributes if not attr.startswith('__')]
all_imported_modules.extend(
[getattr(peft, _import) for _import in imports])
try:
import peft
except: # noqa
pass
else:
attributes = dir(peft)
imports = [
attr for attr in attributes if not attr.startswith('__')
]
all_imported_modules.extend(
[getattr(peft, _import) for _import in imports])
if importlib.util.find_spec('diffusers') is not None:
import diffusers
if importlib.util.find_spec('diffusers') is not None:
try:
import diffusers
except: # noqa
pass
else:
lazy_module = sys.modules['diffusers']
_import_structure = lazy_module._import_structure
for key in _import_structure:
values = _import_structure[key]
for value in values:
if any([name in value
for name in diffusers_include_names]):
try:
module = importlib.import_module(
f'.{key}', diffusers.__name__)
value = getattr(module, value)
all_imported_modules.append(value)
except (ImportError, AttributeError):
pass
if hasattr(lazy_module, '_import_structure'):
_import_structure = lazy_module._import_structure
for key in _import_structure:
values = _import_structure[key]
for value in values:
if any([
name in value
for name in diffusers_include_names
]):
try:
module = importlib.import_module(
f'.{key}', diffusers.__name__)
value = getattr(module, value)
all_imported_modules.append(value)
except (ImportError, AttributeError):
pass
else:
attributes = dir(lazy_module)
imports = [
attr for attr in attributes if not attr.startswith('__')
]
all_imported_modules.extend(
[getattr(lazy_module, _import) for _import in imports])
return all_imported_modules