mirror of
https://github.com/modelscope/modelscope.git
synced 2026-09-02 12:09:36 +02:00
fix: reuse legacy hub cache and use build-time PyPI fallback
Reuse flat/hub legacy caches in download paths. For docker builds, pass PIP_EXTRA_INDEX_URL via ARG and RUN export so PyPI is available during install without persisting it as an image ENV default. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then
|
||||
pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/
|
||||
pip config set install.trusted-host mirrors.aliyun.com
|
||||
pip install -r requirements/tests.txt
|
||||
PIP_EXTRA_INDEX_URL=https://pypi.org/simple pip install -r requirements/tests.txt
|
||||
git config --global --add safe.directory /Maas-lib
|
||||
git config --global user.email tmp
|
||||
git config --global user.name tmp.com
|
||||
|
||||
@@ -3,12 +3,17 @@ FROM {base_image}
|
||||
ARG BASE_IMAGE_TAG={base_image_tag}
|
||||
LABEL modelscope.base_image="vllm/vllm-openai-rocm:${BASE_IMAGE_TAG}"
|
||||
|
||||
# Build-time only (ARG does not persist into the image). Aliyun mirror can lag PyPI (~1h).
|
||||
ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple
|
||||
|
||||
COPY docker/scripts/modelscope_env_init.sh /usr/local/bin/ms_env_init.sh
|
||||
|
||||
ARG CUR_TIME={cur_time}
|
||||
RUN echo "CUR_TIME=${CUR_TIME}" && echo "BASE_IMAGE_TAG=${BASE_IMAGE_TAG}"
|
||||
|
||||
RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \
|
||||
ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple
|
||||
RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
|
||||
pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \
|
||||
pip config set install.trusted-host mirrors.aliyun.com && \
|
||||
cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b {modelscope_branch} --single-branch https://github.com/modelscope/modelscope.git && \
|
||||
cd modelscope && pip install --no-cache-dir . -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
|
||||
|
||||
@@ -5,6 +5,8 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
PIP_RETRIES=10 \
|
||||
SOC_VERSION={soc_version} \
|
||||
CANN_VERSION={cann_version}
|
||||
# Build-time only (ARG does not persist into the image).
|
||||
ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple
|
||||
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
|
||||
@@ -19,12 +21,7 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \
|
||||
pip config set global.extra-index-url "https://pypi.org/simple" && \
|
||||
pip config set install.trusted-host mirrors.aliyun.com && \
|
||||
ARCH=$(uname -m) && \
|
||||
if [ "$ARCH" = "x86_64" ]; then \
|
||||
pip config set global.extra-index-url "https://pypi.org/simple https://download.pytorch.org/whl/cpu/"; \
|
||||
fi
|
||||
pip config set install.trusted-host mirrors.aliyun.com
|
||||
|
||||
{extra_content}
|
||||
# ---------- Install vllm + vllm-ascend ----------
|
||||
@@ -34,6 +31,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
||||
git clone --depth 1 --branch v0.18.0 https://github.com/vllm-project/vllm-ascend.git
|
||||
|
||||
RUN ARCH=$(uname -m) && \
|
||||
export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh && \
|
||||
# Install torch & torch_npu & torchvision
|
||||
@@ -50,7 +48,8 @@ RUN git clone --depth 1 --branch {megatron_branch} https://github.com/NVIDIA/Meg
|
||||
git clone --depth 1 https://github.com/modelscope/mcore-bridge.git /mcore-bridge
|
||||
|
||||
# ---------- Install training-side repositories ----------
|
||||
RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
||||
RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
||||
if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \
|
||||
cd /MindSpeed && pip install --no-cache-dir -e . && \
|
||||
cd /mcore-bridge && pip install --no-cache-dir -e . && \
|
||||
@@ -59,7 +58,8 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
||||
# ---------- Pin torch to the correct version + torch_npu ----------
|
||||
# x86: must force-install the CPU build from pytorch.org/whl/cpu
|
||||
# aarch64: PyPI only provides the CPU build, so install it directly from the Aliyun mirror
|
||||
RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
||||
RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
||||
if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \
|
||||
ARCH=$(uname -m) && \
|
||||
if [ "$ARCH" = "x86_64" ]; then \
|
||||
@@ -83,7 +83,8 @@ ENV PYTHONPATH=/Megatron-LM:${PYTHONPATH}
|
||||
# install dependencies
|
||||
COPY requirements /var/modelscope
|
||||
|
||||
RUN pip uninstall ms-swift modelscope -y && pip install --no-cache-dir pip==23.* -U && \
|
||||
RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
|
||||
pip uninstall ms-swift modelscope -y && pip install --no-cache-dir pip==23.* -U && \
|
||||
if [ "$INSTALL_MS_DEPS" = "True" ]; then \
|
||||
pip install --no-cache-dir omegaconf==2.0.6 && \
|
||||
pip install 'editdistance==0.8.1' && \
|
||||
@@ -109,9 +110,11 @@ fi
|
||||
ARG CUR_TIME={cur_time}
|
||||
RUN echo $CUR_TIME
|
||||
|
||||
RUN pip install --no-cache-dir --no-build-isolation OpenCC
|
||||
RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
|
||||
pip install --no-cache-dir --no-build-isolation OpenCC
|
||||
|
||||
RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
||||
RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
||||
if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \
|
||||
pip install --no-cache-dir -U funasr scikit-learn && \
|
||||
pip install --no-cache-dir -U qwen_vl_utils qwen_omni_utils librosa 'timm>=0.9.0' transformers accelerate peft trl safetensors && \
|
||||
@@ -128,6 +131,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
||||
|
||||
# ---------- Reinstall triton-ascend for the selected CANN version ----------
|
||||
RUN set -eux; \
|
||||
export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}"; \
|
||||
pip uninstall -y triton || true; \
|
||||
pip uninstall -y triton-ascend || true; \
|
||||
case "${CANN_VERSION}" in \
|
||||
|
||||
@@ -3,6 +3,8 @@ FROM {base_image}
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
ENV TZ=Asia/Shanghai
|
||||
ENV arch=x86_64
|
||||
# Build-time only (ARG does not persist into the image). Aliyun mirror can lag PyPI (~1h).
|
||||
ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple
|
||||
|
||||
COPY docker/scripts/modelscope_env_init.sh /usr/local/bin/ms_env_init.sh
|
||||
RUN apt-get update && \
|
||||
@@ -21,7 +23,9 @@ ARG IMAGE_TYPE={image_type}
|
||||
# install dependencies
|
||||
COPY requirements /var/modelscope
|
||||
|
||||
RUN pip uninstall ms-swift modelscope -y && pip --no-cache-dir install pip==23.* -U && \
|
||||
ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple
|
||||
RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
|
||||
pip uninstall ms-swift modelscope -y && pip --no-cache-dir install pip==23.* -U && \
|
||||
if [ "$INSTALL_MS_DEPS" = "True" ]; then \
|
||||
pip --no-cache-dir install omegaconf==2.0.6 && \
|
||||
pip install 'editdistance==0.8.1' && \
|
||||
@@ -47,7 +51,9 @@ fi
|
||||
ARG CUR_TIME={cur_time}
|
||||
RUN echo $CUR_TIME
|
||||
|
||||
RUN bash /tmp/install.sh {version_args} && \
|
||||
ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple
|
||||
RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
|
||||
bash /tmp/install.sh {version_args} && \
|
||||
pip install --no-cache-dir -U funasr scikit-learn && \
|
||||
pip install --no-cache-dir -U qwen_vl_utils qwen_omni_utils librosa timm transformers accelerate peft trl safetensors && \
|
||||
cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b {swift_branch} --single-branch https://github.com/modelscope/ms-swift.git && \
|
||||
@@ -67,7 +73,9 @@ RUN bash /tmp/install.sh {version_args} && \
|
||||
cp /tmp/resources/ubuntu2204.aliyun /etc/apt/sources.list
|
||||
|
||||
|
||||
RUN if [ "$IMAGE_TYPE" = "gpu" ]; then \
|
||||
ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple
|
||||
RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
|
||||
if [ "$IMAGE_TYPE" = "gpu" ]; then \
|
||||
pip install --no-cache-dir math_verify "gradio<5.33" "deepspeed<0.19" ray -U && \
|
||||
pip install --no-cache-dir mcore-bridge -i https://pypi.org/simple/ -U && \
|
||||
pip install --no-cache-dir liger_kernel wandb swanlab nvitop pre-commit "transformers<5.9" "trl<1.0" "peft<0.20" huggingface-hub -U && \
|
||||
|
||||
@@ -372,7 +372,8 @@ class StableGPUImageBuilder(Builder):
|
||||
extra_content = extra_content.replace('{python_version}',
|
||||
self.args.python_version)
|
||||
extra_content += """
|
||||
RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy
|
||||
RUN export PIP_EXTRA_INDEX_URL=https://pypi.org/simple && \
|
||||
pip install --no-cache-dir -U icecream soundfile pybind11 py-spy
|
||||
"""
|
||||
version_args = (
|
||||
f'{self.args.torch_version} {self.args.torchvision_version} {self.args.torchaudio_version} '
|
||||
@@ -433,7 +434,8 @@ class LatestGPUImageBuilder(StableGPUImageBuilder):
|
||||
extra_content = extra_content.replace('{python_version}',
|
||||
self.args.python_version)
|
||||
extra_content += """
|
||||
RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy
|
||||
RUN export PIP_EXTRA_INDEX_URL=https://pypi.org/simple && \
|
||||
pip install --no-cache-dir -U icecream soundfile pybind11 py-spy
|
||||
"""
|
||||
version_args = (
|
||||
f'{self.args.torch_version} {self.args.torchvision_version} {self.args.torchaudio_version} '
|
||||
@@ -886,7 +888,8 @@ class AscendImageBuilder(StableGPUImageBuilder):
|
||||
|
||||
def generate_dockerfile(self) -> str:
|
||||
extra_content = """
|
||||
RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy
|
||||
RUN export PIP_EXTRA_INDEX_URL=https://pypi.org/simple && \
|
||||
pip install --no-cache-dir -U icecream soundfile pybind11 py-spy
|
||||
"""
|
||||
with open('docker/Dockerfile.ascend', 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
@@ -18,7 +18,8 @@ from typing import Dict, List, Optional, Type
|
||||
|
||||
import requests
|
||||
# --- Hub file downloads (delegated) ---
|
||||
from modelscope_hub.compat import dataset_file_download # noqa: E402,F401
|
||||
from modelscope_hub.compat.file_download import \
|
||||
dataset_file_download as _compat_dataset_file_download
|
||||
from modelscope_hub.compat.file_download import \
|
||||
model_file_download as _compat_model_file_download
|
||||
from requests.adapters import Retry
|
||||
@@ -31,7 +32,7 @@ from modelscope.hub.constants import (API_FILE_DOWNLOAD_CHUNK_SIZE,
|
||||
from modelscope.utils.logger import get_logger
|
||||
from .callback import ProgressCallback, TqdmCallback
|
||||
from .errors import FileDownloadError
|
||||
from .utils.utils import get_endpoint
|
||||
from .utils.utils import find_reusable_legacy_repo_dir, get_endpoint
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
@@ -77,6 +78,9 @@ def model_file_download(
|
||||
revision = detail.get('Revision')
|
||||
except Exception:
|
||||
pass
|
||||
if local_dir is None:
|
||||
local_dir = find_reusable_legacy_repo_dir(
|
||||
model_id, repo_type='model', cache_dir=cache_dir)
|
||||
return _compat_model_file_download(
|
||||
model_id,
|
||||
file_path,
|
||||
@@ -91,6 +95,37 @@ def model_file_download(
|
||||
)
|
||||
|
||||
|
||||
def dataset_file_download(
|
||||
dataset_id: str,
|
||||
file_path: str,
|
||||
*,
|
||||
cache_dir: str = None,
|
||||
local_dir: str = None,
|
||||
revision: str = None,
|
||||
cookies: dict = None,
|
||||
token: str = None,
|
||||
endpoint: str = None,
|
||||
local_files_only: bool = False,
|
||||
user_agent=None,
|
||||
) -> str:
|
||||
"""Download a single dataset file, reusing flat/hub legacy caches when present."""
|
||||
if local_dir is None:
|
||||
local_dir = find_reusable_legacy_repo_dir(
|
||||
dataset_id, repo_type='dataset', cache_dir=cache_dir)
|
||||
return _compat_dataset_file_download(
|
||||
dataset_id,
|
||||
file_path,
|
||||
cache_dir=cache_dir,
|
||||
local_dir=local_dir,
|
||||
revision=revision,
|
||||
cookies=cookies,
|
||||
token=token,
|
||||
endpoint=endpoint,
|
||||
local_files_only=local_files_only,
|
||||
user_agent=user_agent,
|
||||
)
|
||||
|
||||
|
||||
# --- Direct HTTP downloads (retained - non-Hub API) ---
|
||||
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ from modelscope_hub.compat.snapshot_download import \
|
||||
from modelscope_hub.compat.snapshot_download import \
|
||||
snapshot_download as _compat_snapshot_download
|
||||
|
||||
from modelscope.hub.utils.utils import find_reusable_legacy_repo_dir
|
||||
from modelscope.utils.logger import get_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -83,10 +84,16 @@ def snapshot_download(
|
||||
(not instances), each instantiated per file to report download progress.
|
||||
"""
|
||||
_warn_if_legacy_cache_detection_unavailable()
|
||||
effective_id = repo_id or model_id
|
||||
effective_type = repo_type or 'model'
|
||||
cache_dir_str = str(cache_dir) if cache_dir is not None else None
|
||||
if local_dir is None and effective_id is not None:
|
||||
local_dir = find_reusable_legacy_repo_dir(
|
||||
effective_id, repo_type=effective_type, cache_dir=cache_dir_str)
|
||||
return _compat_snapshot_download(
|
||||
model_id=model_id,
|
||||
revision=revision,
|
||||
cache_dir=str(cache_dir) if cache_dir is not None else None,
|
||||
cache_dir=cache_dir_str,
|
||||
local_dir=local_dir,
|
||||
allow_file_pattern=allow_file_pattern,
|
||||
ignore_file_pattern=ignore_file_pattern,
|
||||
@@ -123,10 +130,14 @@ def dataset_snapshot_download(
|
||||
"""Download a dataset repo snapshot (legacy positional-arg signature)."""
|
||||
_warn_if_legacy_cache_detection_unavailable()
|
||||
effective_id = dataset_id or repo_id
|
||||
cache_dir_str = str(cache_dir) if cache_dir is not None else None
|
||||
if local_dir is None and effective_id is not None:
|
||||
local_dir = find_reusable_legacy_repo_dir(
|
||||
effective_id, repo_type='dataset', cache_dir=cache_dir_str)
|
||||
return _compat_dataset_snapshot_download(
|
||||
dataset_id=effective_id,
|
||||
revision=revision,
|
||||
cache_dir=str(cache_dir) if cache_dir is not None else None,
|
||||
cache_dir=cache_dir_str,
|
||||
local_dir=local_dir,
|
||||
allow_file_pattern=allow_file_pattern,
|
||||
ignore_file_pattern=ignore_file_pattern,
|
||||
|
||||
@@ -194,6 +194,78 @@ def get_cache_dir(model_id: Optional[str] = None):
|
||||
base_path, model_id + '/')
|
||||
|
||||
|
||||
def _modelscope_hub_cache_root() -> Path:
|
||||
"""Cache root used by ``modelscope_hub`` downloads (not SDK ``.../hub``)."""
|
||||
env = os.environ.get('MODELSCOPE_CACHE')
|
||||
if env:
|
||||
return Path(env).expanduser()
|
||||
return Path.home() / '.cache' / 'modelscope'
|
||||
|
||||
|
||||
def find_reusable_legacy_repo_dir(
|
||||
repo_id: str,
|
||||
repo_type: str = 'model',
|
||||
cache_dir: Optional[Union[str, Path]] = None,
|
||||
) -> Optional[str]:
|
||||
"""Find old on-disk cache layouts that ``modelscope_hub`` download misses.
|
||||
|
||||
``modelscope_hub`` reuses ``{cache}/{type}s/{owner}/{safe_name}/`` (dots in
|
||||
``name`` replaced by ``___``) and writes to
|
||||
``{cache}/{type}s/{owner}--{name}/snapshots/{rev}/``. Older SDKs also
|
||||
stored repos at:
|
||||
|
||||
- ``{cache}/{owner}/{name}/`` (flat, when ``MODELSCOPE_CACHE`` was set)
|
||||
- ``{cache}/hub/{owner}/{name}/`` (pre-``models/`` restructuring)
|
||||
- ``{cache}/{type}s/{owner}/{name}/`` (unsafed name; hub only checks
|
||||
``safe_name``)
|
||||
|
||||
Returns a non-empty legacy path only when the layouts hub already handles
|
||||
are absent, so callers can pass it as ``local_dir`` and avoid re-download.
|
||||
"""
|
||||
if not repo_id or '/' not in repo_id:
|
||||
return None
|
||||
|
||||
base = Path(cache_dir).expanduser() if cache_dir is not None else \
|
||||
_modelscope_hub_cache_root()
|
||||
segment = f'{repo_type}s' if not repo_type.endswith('s') else repo_type
|
||||
owner, name = repo_id.split('/', 1)
|
||||
safe_name = name.replace('.', '___')
|
||||
safe_id = repo_id.replace('/', '--')
|
||||
|
||||
# Layouts already handled by modelscope_hub — do not override.
|
||||
hub_known = [
|
||||
base / segment / safe_id,
|
||||
base / segment / owner / safe_name,
|
||||
]
|
||||
for path in hub_known:
|
||||
if _non_empty_dir(path):
|
||||
return None
|
||||
|
||||
# Layouts hub download does not probe today.
|
||||
legacy_candidates = [
|
||||
base / owner / name,
|
||||
base / owner / safe_name,
|
||||
base / 'hub' / owner / name,
|
||||
base / 'hub' / owner / safe_name,
|
||||
base / segment / owner / name,
|
||||
]
|
||||
for path in legacy_candidates:
|
||||
if _non_empty_dir(path):
|
||||
logger.info(
|
||||
'Found legacy cache at %s for %s, reusing.', path, repo_id)
|
||||
return str(path)
|
||||
return None
|
||||
|
||||
|
||||
def _non_empty_dir(path: Path) -> bool:
|
||||
if not path.is_dir():
|
||||
return False
|
||||
try:
|
||||
return any(path.iterdir())
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def get_release_datetime():
|
||||
if MODELSCOPE_SDK_DEBUG in os.environ:
|
||||
rt = int(round(datetime.now().timestamp()))
|
||||
|
||||
102
tests/hub/test_legacy_cache_reuse.py
Normal file
102
tests/hub/test_legacy_cache_reuse.py
Normal file
@@ -0,0 +1,102 @@
|
||||
# Copyright (c) Alibaba, Inc. and its affiliates.
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from modelscope.hub.utils.utils import find_reusable_legacy_repo_dir
|
||||
|
||||
|
||||
class LegacyCacheReuseTest(unittest.TestCase):
|
||||
"""Old flat/hub cache layouts should be reusable without re-download."""
|
||||
|
||||
def setUp(self):
|
||||
self._tmpdir = tempfile.TemporaryDirectory()
|
||||
self.cache = Path(self._tmpdir.name)
|
||||
self.model_id = 'iic/nlp_xlmr_named-entity-recognition_eng-ecommerce-query'
|
||||
self.owner, self.name = self.model_id.split('/', 1)
|
||||
|
||||
def tearDown(self):
|
||||
self._tmpdir.cleanup()
|
||||
|
||||
def _touch_model_dir(self, path: Path):
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
(path / 'configuration.json').write_text('{}', encoding='utf-8')
|
||||
|
||||
def test_finds_flat_legacy_cache(self):
|
||||
legacy = self.cache / self.owner / self.name
|
||||
self._touch_model_dir(legacy)
|
||||
found = find_reusable_legacy_repo_dir(
|
||||
self.model_id, cache_dir=self.cache)
|
||||
self.assertEqual(found, str(legacy))
|
||||
|
||||
def test_finds_hub_legacy_cache(self):
|
||||
legacy = self.cache / 'hub' / self.owner / self.name
|
||||
self._touch_model_dir(legacy)
|
||||
found = find_reusable_legacy_repo_dir(
|
||||
self.model_id, cache_dir=self.cache)
|
||||
self.assertEqual(found, str(legacy))
|
||||
|
||||
def test_reuses_unsafed_models_slash_layout(self):
|
||||
# Hub only probes safe_name (dots -> ___); unsafed path is reusable.
|
||||
dotted_id = 'org/model.with.dots'
|
||||
owner, name = dotted_id.split('/', 1)
|
||||
slash = self.cache / 'models' / owner / name
|
||||
self._touch_model_dir(slash)
|
||||
found = find_reusable_legacy_repo_dir(dotted_id, cache_dir=self.cache)
|
||||
self.assertEqual(found, str(slash))
|
||||
|
||||
def test_prefers_hub_known_safe_slash_layout(self):
|
||||
dotted_id = 'org/model.with.dots'
|
||||
owner, name = dotted_id.split('/', 1)
|
||||
safe = self.cache / 'models' / owner / name.replace('.', '___')
|
||||
flat = self.cache / owner / name
|
||||
self._touch_model_dir(safe)
|
||||
self._touch_model_dir(flat)
|
||||
found = find_reusable_legacy_repo_dir(dotted_id, cache_dir=self.cache)
|
||||
self.assertIsNone(found)
|
||||
|
||||
def test_prefers_hub_known_owner_dash_layout(self):
|
||||
modern = self.cache / 'models' / self.model_id.replace('/', '--')
|
||||
flat = self.cache / self.owner / self.name
|
||||
self._touch_model_dir(modern)
|
||||
self._touch_model_dir(flat)
|
||||
found = find_reusable_legacy_repo_dir(
|
||||
self.model_id, cache_dir=self.cache)
|
||||
self.assertIsNone(found)
|
||||
|
||||
def test_empty_legacy_dir_ignored(self):
|
||||
(self.cache / self.owner / self.name).mkdir(parents=True)
|
||||
found = find_reusable_legacy_repo_dir(
|
||||
self.model_id, cache_dir=self.cache)
|
||||
self.assertIsNone(found)
|
||||
|
||||
def test_uses_modelscope_cache_env(self):
|
||||
legacy = self.cache / self.owner / self.name
|
||||
self._touch_model_dir(legacy)
|
||||
with mock.patch.dict(os.environ, {'MODELSCOPE_CACHE': str(self.cache)}):
|
||||
found = find_reusable_legacy_repo_dir(self.model_id)
|
||||
self.assertEqual(found, str(legacy))
|
||||
|
||||
def test_default_root_matches_hub_not_sdk_hub_suffix(self):
|
||||
# Without MODELSCOPE_CACHE, hub uses ~/.cache/modelscope (no /hub).
|
||||
modern = (
|
||||
Path.home() / '.cache' / 'modelscope' / 'models'
|
||||
/ self.model_id.replace('/', '--'))
|
||||
# Do not create real home dirs; patch the hub root helper instead.
|
||||
with mock.patch(
|
||||
'modelscope.hub.utils.utils._modelscope_hub_cache_root',
|
||||
return_value=self.cache):
|
||||
modern_under_test = (
|
||||
self.cache / 'models' / self.model_id.replace('/', '--'))
|
||||
flat = self.cache / self.owner / self.name
|
||||
self._touch_model_dir(modern_under_test)
|
||||
self._touch_model_dir(flat)
|
||||
found = find_reusable_legacy_repo_dir(self.model_id)
|
||||
self.assertIsNone(found)
|
||||
self.assertFalse(modern.exists()) # we never touched real home cache
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user