fix: reuse legacy hub cache and use build-time PyPI fallback

Reuse flat/hub legacy caches in download paths. For docker builds, pass
PIP_EXTRA_INDEX_URL via ARG and RUN export so PyPI is available during
install without persisting it as an image ENV default.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
suluyan
2026-07-21 20:01:05 +08:00
parent 9ab427b29a
commit cfe1f4190c
9 changed files with 263 additions and 23 deletions

View File

@@ -1,7 +1,7 @@
if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then
pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/
pip config set install.trusted-host mirrors.aliyun.com
pip install -r requirements/tests.txt
PIP_EXTRA_INDEX_URL=https://pypi.org/simple pip install -r requirements/tests.txt
git config --global --add safe.directory /Maas-lib
git config --global user.email tmp
git config --global user.name tmp.com

View File

@@ -3,12 +3,17 @@ FROM {base_image}
ARG BASE_IMAGE_TAG={base_image_tag}
LABEL modelscope.base_image="vllm/vllm-openai-rocm:${BASE_IMAGE_TAG}"
# Build-time only (ARG does not persist into the image). Aliyun mirror can lag PyPI (~1h).
ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple
COPY docker/scripts/modelscope_env_init.sh /usr/local/bin/ms_env_init.sh
ARG CUR_TIME={cur_time}
RUN echo "CUR_TIME=${CUR_TIME}" && echo "BASE_IMAGE_TAG=${BASE_IMAGE_TAG}"
RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \
ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple
RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \
pip config set install.trusted-host mirrors.aliyun.com && \
cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b {modelscope_branch} --single-branch https://github.com/modelscope/modelscope.git && \
cd modelscope && pip install --no-cache-dir . -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \

View File

@@ -5,6 +5,8 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \
PIP_RETRIES=10 \
SOC_VERSION={soc_version} \
CANN_VERSION={cann_version}
# Build-time only (ARG does not persist into the image).
ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple
SHELL ["/bin/bash", "-c"]
@@ -19,12 +21,7 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean && \
rm -rf /var/lib/apt/lists/*
RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \
pip config set global.extra-index-url "https://pypi.org/simple" && \
pip config set install.trusted-host mirrors.aliyun.com && \
ARCH=$(uname -m) && \
if [ "$ARCH" = "x86_64" ]; then \
pip config set global.extra-index-url "https://pypi.org/simple https://download.pytorch.org/whl/cpu/"; \
fi
pip config set install.trusted-host mirrors.aliyun.com
{extra_content}
# ---------- Install vllm + vllm-ascend ----------
@@ -34,6 +31,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
git clone --depth 1 --branch v0.18.0 https://github.com/vllm-project/vllm-ascend.git
RUN ARCH=$(uname -m) && \
export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
source /usr/local/Ascend/nnal/atb/set_env.sh && \
# Install torch & torch_npu & torchvision
@@ -50,7 +48,8 @@ RUN git clone --depth 1 --branch {megatron_branch} https://github.com/NVIDIA/Meg
git clone --depth 1 https://github.com/modelscope/mcore-bridge.git /mcore-bridge
# ---------- Install training-side repositories ----------
RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \
cd /MindSpeed && pip install --no-cache-dir -e . && \
cd /mcore-bridge && pip install --no-cache-dir -e . && \
@@ -59,7 +58,8 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
# ---------- Pin torch to the correct version + torch_npu ----------
# x86: must force-install the CPU build from pytorch.org/whl/cpu
# aarch64: PyPI only provides the CPU build, so install it directly from the Aliyun mirror
RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \
ARCH=$(uname -m) && \
if [ "$ARCH" = "x86_64" ]; then \
@@ -83,7 +83,8 @@ ENV PYTHONPATH=/Megatron-LM:${PYTHONPATH}
# install dependencies
COPY requirements /var/modelscope
RUN pip uninstall ms-swift modelscope -y && pip install --no-cache-dir pip==23.* -U && \
RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
pip uninstall ms-swift modelscope -y && pip install --no-cache-dir pip==23.* -U && \
if [ "$INSTALL_MS_DEPS" = "True" ]; then \
pip install --no-cache-dir omegaconf==2.0.6 && \
pip install 'editdistance==0.8.1' && \
@@ -109,9 +110,11 @@ fi
ARG CUR_TIME={cur_time}
RUN echo $CUR_TIME
RUN pip install --no-cache-dir --no-build-isolation OpenCC
RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
pip install --no-cache-dir --no-build-isolation OpenCC
RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \
pip install --no-cache-dir -U funasr scikit-learn && \
pip install --no-cache-dir -U qwen_vl_utils qwen_omni_utils librosa 'timm>=0.9.0' transformers accelerate peft trl safetensors && \
@@ -128,6 +131,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
# ---------- Reinstall triton-ascend for the selected CANN version ----------
RUN set -eux; \
export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}"; \
pip uninstall -y triton || true; \
pip uninstall -y triton-ascend || true; \
case "${CANN_VERSION}" in \

View File

@@ -3,6 +3,8 @@ FROM {base_image}
ARG DEBIAN_FRONTEND=noninteractive
ENV TZ=Asia/Shanghai
ENV arch=x86_64
# Build-time only (ARG does not persist into the image). Aliyun mirror can lag PyPI (~1h).
ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple
COPY docker/scripts/modelscope_env_init.sh /usr/local/bin/ms_env_init.sh
RUN apt-get update && \
@@ -21,7 +23,9 @@ ARG IMAGE_TYPE={image_type}
# install dependencies
COPY requirements /var/modelscope
RUN pip uninstall ms-swift modelscope -y && pip --no-cache-dir install pip==23.* -U && \
ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple
RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
pip uninstall ms-swift modelscope -y && pip --no-cache-dir install pip==23.* -U && \
if [ "$INSTALL_MS_DEPS" = "True" ]; then \
pip --no-cache-dir install omegaconf==2.0.6 && \
pip install 'editdistance==0.8.1' && \
@@ -47,7 +51,9 @@ fi
ARG CUR_TIME={cur_time}
RUN echo $CUR_TIME
RUN bash /tmp/install.sh {version_args} && \
ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple
RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
bash /tmp/install.sh {version_args} && \
pip install --no-cache-dir -U funasr scikit-learn && \
pip install --no-cache-dir -U qwen_vl_utils qwen_omni_utils librosa timm transformers accelerate peft trl safetensors && \
cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b {swift_branch} --single-branch https://github.com/modelscope/ms-swift.git && \
@@ -67,7 +73,9 @@ RUN bash /tmp/install.sh {version_args} && \
cp /tmp/resources/ubuntu2204.aliyun /etc/apt/sources.list
RUN if [ "$IMAGE_TYPE" = "gpu" ]; then \
ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple
RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
if [ "$IMAGE_TYPE" = "gpu" ]; then \
pip install --no-cache-dir math_verify "gradio<5.33" "deepspeed<0.19" ray -U && \
pip install --no-cache-dir mcore-bridge -i https://pypi.org/simple/ -U && \
pip install --no-cache-dir liger_kernel wandb swanlab nvitop pre-commit "transformers<5.9" "trl<1.0" "peft<0.20" huggingface-hub -U && \

View File

@@ -372,7 +372,8 @@ class StableGPUImageBuilder(Builder):
extra_content = extra_content.replace('{python_version}',
self.args.python_version)
extra_content += """
RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy
RUN export PIP_EXTRA_INDEX_URL=https://pypi.org/simple && \
pip install --no-cache-dir -U icecream soundfile pybind11 py-spy
"""
version_args = (
f'{self.args.torch_version} {self.args.torchvision_version} {self.args.torchaudio_version} '
@@ -433,7 +434,8 @@ class LatestGPUImageBuilder(StableGPUImageBuilder):
extra_content = extra_content.replace('{python_version}',
self.args.python_version)
extra_content += """
RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy
RUN export PIP_EXTRA_INDEX_URL=https://pypi.org/simple && \
pip install --no-cache-dir -U icecream soundfile pybind11 py-spy
"""
version_args = (
f'{self.args.torch_version} {self.args.torchvision_version} {self.args.torchaudio_version} '
@@ -886,7 +888,8 @@ class AscendImageBuilder(StableGPUImageBuilder):
def generate_dockerfile(self) -> str:
extra_content = """
RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy
RUN export PIP_EXTRA_INDEX_URL=https://pypi.org/simple && \
pip install --no-cache-dir -U icecream soundfile pybind11 py-spy
"""
with open('docker/Dockerfile.ascend', 'r') as f:
content = f.read()

View File

@@ -18,7 +18,8 @@ from typing import Dict, List, Optional, Type
import requests
# --- Hub file downloads (delegated) ---
from modelscope_hub.compat import dataset_file_download # noqa: E402,F401
from modelscope_hub.compat.file_download import \
dataset_file_download as _compat_dataset_file_download
from modelscope_hub.compat.file_download import \
model_file_download as _compat_model_file_download
from requests.adapters import Retry
@@ -31,7 +32,7 @@ from modelscope.hub.constants import (API_FILE_DOWNLOAD_CHUNK_SIZE,
from modelscope.utils.logger import get_logger
from .callback import ProgressCallback, TqdmCallback
from .errors import FileDownloadError
from .utils.utils import get_endpoint
from .utils.utils import find_reusable_legacy_repo_dir, get_endpoint
logger = get_logger()
@@ -77,6 +78,9 @@ def model_file_download(
revision = detail.get('Revision')
except Exception:
pass
if local_dir is None:
local_dir = find_reusable_legacy_repo_dir(
model_id, repo_type='model', cache_dir=cache_dir)
return _compat_model_file_download(
model_id,
file_path,
@@ -91,6 +95,37 @@ def model_file_download(
)
def dataset_file_download(
dataset_id: str,
file_path: str,
*,
cache_dir: str = None,
local_dir: str = None,
revision: str = None,
cookies: dict = None,
token: str = None,
endpoint: str = None,
local_files_only: bool = False,
user_agent=None,
) -> str:
"""Download a single dataset file, reusing flat/hub legacy caches when present."""
if local_dir is None:
local_dir = find_reusable_legacy_repo_dir(
dataset_id, repo_type='dataset', cache_dir=cache_dir)
return _compat_dataset_file_download(
dataset_id,
file_path,
cache_dir=cache_dir,
local_dir=local_dir,
revision=revision,
cookies=cookies,
token=token,
endpoint=endpoint,
local_files_only=local_files_only,
user_agent=user_agent,
)
# --- Direct HTTP downloads (retained - non-Hub API) ---

View File

@@ -13,6 +13,7 @@ from modelscope_hub.compat.snapshot_download import \
from modelscope_hub.compat.snapshot_download import \
snapshot_download as _compat_snapshot_download
from modelscope.hub.utils.utils import find_reusable_legacy_repo_dir
from modelscope.utils.logger import get_logger
if TYPE_CHECKING:
@@ -83,10 +84,16 @@ def snapshot_download(
(not instances), each instantiated per file to report download progress.
"""
_warn_if_legacy_cache_detection_unavailable()
effective_id = repo_id or model_id
effective_type = repo_type or 'model'
cache_dir_str = str(cache_dir) if cache_dir is not None else None
if local_dir is None and effective_id is not None:
local_dir = find_reusable_legacy_repo_dir(
effective_id, repo_type=effective_type, cache_dir=cache_dir_str)
return _compat_snapshot_download(
model_id=model_id,
revision=revision,
cache_dir=str(cache_dir) if cache_dir is not None else None,
cache_dir=cache_dir_str,
local_dir=local_dir,
allow_file_pattern=allow_file_pattern,
ignore_file_pattern=ignore_file_pattern,
@@ -123,10 +130,14 @@ def dataset_snapshot_download(
"""Download a dataset repo snapshot (legacy positional-arg signature)."""
_warn_if_legacy_cache_detection_unavailable()
effective_id = dataset_id or repo_id
cache_dir_str = str(cache_dir) if cache_dir is not None else None
if local_dir is None and effective_id is not None:
local_dir = find_reusable_legacy_repo_dir(
effective_id, repo_type='dataset', cache_dir=cache_dir_str)
return _compat_dataset_snapshot_download(
dataset_id=effective_id,
revision=revision,
cache_dir=str(cache_dir) if cache_dir is not None else None,
cache_dir=cache_dir_str,
local_dir=local_dir,
allow_file_pattern=allow_file_pattern,
ignore_file_pattern=ignore_file_pattern,

View File

@@ -194,6 +194,78 @@ def get_cache_dir(model_id: Optional[str] = None):
base_path, model_id + '/')
def _modelscope_hub_cache_root() -> Path:
"""Cache root used by ``modelscope_hub`` downloads (not SDK ``.../hub``)."""
env = os.environ.get('MODELSCOPE_CACHE')
if env:
return Path(env).expanduser()
return Path.home() / '.cache' / 'modelscope'
def find_reusable_legacy_repo_dir(
repo_id: str,
repo_type: str = 'model',
cache_dir: Optional[Union[str, Path]] = None,
) -> Optional[str]:
"""Find old on-disk cache layouts that ``modelscope_hub`` download misses.
``modelscope_hub`` reuses ``{cache}/{type}s/{owner}/{safe_name}/`` (dots in
``name`` replaced by ``___``) and writes to
``{cache}/{type}s/{owner}--{name}/snapshots/{rev}/``. Older SDKs also
stored repos at:
- ``{cache}/{owner}/{name}/`` (flat, when ``MODELSCOPE_CACHE`` was set)
- ``{cache}/hub/{owner}/{name}/`` (pre-``models/`` restructuring)
- ``{cache}/{type}s/{owner}/{name}/`` (unsafed name; hub only checks
``safe_name``)
Returns a non-empty legacy path only when the layouts hub already handles
are absent, so callers can pass it as ``local_dir`` and avoid re-download.
"""
if not repo_id or '/' not in repo_id:
return None
base = Path(cache_dir).expanduser() if cache_dir is not None else \
_modelscope_hub_cache_root()
segment = f'{repo_type}s' if not repo_type.endswith('s') else repo_type
owner, name = repo_id.split('/', 1)
safe_name = name.replace('.', '___')
safe_id = repo_id.replace('/', '--')
# Layouts already handled by modelscope_hub — do not override.
hub_known = [
base / segment / safe_id,
base / segment / owner / safe_name,
]
for path in hub_known:
if _non_empty_dir(path):
return None
# Layouts hub download does not probe today.
legacy_candidates = [
base / owner / name,
base / owner / safe_name,
base / 'hub' / owner / name,
base / 'hub' / owner / safe_name,
base / segment / owner / name,
]
for path in legacy_candidates:
if _non_empty_dir(path):
logger.info(
'Found legacy cache at %s for %s, reusing.', path, repo_id)
return str(path)
return None
def _non_empty_dir(path: Path) -> bool:
if not path.is_dir():
return False
try:
return any(path.iterdir())
except OSError:
return False
def get_release_datetime():
if MODELSCOPE_SDK_DEBUG in os.environ:
rt = int(round(datetime.now().timestamp()))

View File

@@ -0,0 +1,102 @@
# Copyright (c) Alibaba, Inc. and its affiliates.
import os
import tempfile
import unittest
from pathlib import Path
from unittest import mock
from modelscope.hub.utils.utils import find_reusable_legacy_repo_dir
class LegacyCacheReuseTest(unittest.TestCase):
"""Old flat/hub cache layouts should be reusable without re-download."""
def setUp(self):
self._tmpdir = tempfile.TemporaryDirectory()
self.cache = Path(self._tmpdir.name)
self.model_id = 'iic/nlp_xlmr_named-entity-recognition_eng-ecommerce-query'
self.owner, self.name = self.model_id.split('/', 1)
def tearDown(self):
self._tmpdir.cleanup()
def _touch_model_dir(self, path: Path):
path.mkdir(parents=True, exist_ok=True)
(path / 'configuration.json').write_text('{}', encoding='utf-8')
def test_finds_flat_legacy_cache(self):
legacy = self.cache / self.owner / self.name
self._touch_model_dir(legacy)
found = find_reusable_legacy_repo_dir(
self.model_id, cache_dir=self.cache)
self.assertEqual(found, str(legacy))
def test_finds_hub_legacy_cache(self):
legacy = self.cache / 'hub' / self.owner / self.name
self._touch_model_dir(legacy)
found = find_reusable_legacy_repo_dir(
self.model_id, cache_dir=self.cache)
self.assertEqual(found, str(legacy))
def test_reuses_unsafed_models_slash_layout(self):
# Hub only probes safe_name (dots -> ___); unsafed path is reusable.
dotted_id = 'org/model.with.dots'
owner, name = dotted_id.split('/', 1)
slash = self.cache / 'models' / owner / name
self._touch_model_dir(slash)
found = find_reusable_legacy_repo_dir(dotted_id, cache_dir=self.cache)
self.assertEqual(found, str(slash))
def test_prefers_hub_known_safe_slash_layout(self):
dotted_id = 'org/model.with.dots'
owner, name = dotted_id.split('/', 1)
safe = self.cache / 'models' / owner / name.replace('.', '___')
flat = self.cache / owner / name
self._touch_model_dir(safe)
self._touch_model_dir(flat)
found = find_reusable_legacy_repo_dir(dotted_id, cache_dir=self.cache)
self.assertIsNone(found)
def test_prefers_hub_known_owner_dash_layout(self):
modern = self.cache / 'models' / self.model_id.replace('/', '--')
flat = self.cache / self.owner / self.name
self._touch_model_dir(modern)
self._touch_model_dir(flat)
found = find_reusable_legacy_repo_dir(
self.model_id, cache_dir=self.cache)
self.assertIsNone(found)
def test_empty_legacy_dir_ignored(self):
(self.cache / self.owner / self.name).mkdir(parents=True)
found = find_reusable_legacy_repo_dir(
self.model_id, cache_dir=self.cache)
self.assertIsNone(found)
def test_uses_modelscope_cache_env(self):
legacy = self.cache / self.owner / self.name
self._touch_model_dir(legacy)
with mock.patch.dict(os.environ, {'MODELSCOPE_CACHE': str(self.cache)}):
found = find_reusable_legacy_repo_dir(self.model_id)
self.assertEqual(found, str(legacy))
def test_default_root_matches_hub_not_sdk_hub_suffix(self):
# Without MODELSCOPE_CACHE, hub uses ~/.cache/modelscope (no /hub).
modern = (
Path.home() / '.cache' / 'modelscope' / 'models'
/ self.model_id.replace('/', '--'))
# Do not create real home dirs; patch the hub root helper instead.
with mock.patch(
'modelscope.hub.utils.utils._modelscope_hub_cache_root',
return_value=self.cache):
modern_under_test = (
self.cache / 'models' / self.model_id.replace('/', '--'))
flat = self.cache / self.owner / self.name
self._touch_model_dir(modern_under_test)
self._touch_model_dir(flat)
found = find_reusable_legacy_repo_dir(self.model_id)
self.assertIsNone(found)
self.assertFalse(modern.exists()) # we never touched real home cache
if __name__ == '__main__':
unittest.main()