Backport/release 1.39 to master (#1767)

This commit is contained in:
suluyana
2026-08-03 16:16:54 +08:00
committed by GitHub
parent 2905da1220
commit 0fd381159a
20 changed files with 1283 additions and 71 deletions

View File

@@ -1,8 +1,7 @@
if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then
pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/ pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/
pip config set global.extra-index-url https://pypi.org/simple/
pip config set install.trusted-host mirrors.aliyun.com pip config set install.trusted-host mirrors.aliyun.com
pip install -r requirements/tests.txt PIP_EXTRA_INDEX_URL=https://pypi.org/simple pip install -r requirements/tests.txt
git config --global --add safe.directory /Maas-lib git config --global --add safe.directory /Maas-lib
git config --global user.email tmp git config --global user.email tmp
git config --global user.name tmp.com git config --global user.name tmp.com

View File

@@ -11,7 +11,7 @@ on:
description: 'ModelScope branch to build from(release/x.xx)' description: 'ModelScope branch to build from(release/x.xx)'
required: true required: true
image_type: image_type:
description: 'The image type to build(base/old/stable/latest)' description: 'The image type to build(base/old/stable/latest/amd/ascend)'
required: true required: true
modelscope_version: modelscope_version:
description: 'ModelScope version to use(x.xx.x)' description: 'ModelScope version to use(x.xx.x)'

26
docker/Dockerfile.amd Normal file
View File

@@ -0,0 +1,26 @@
FROM {base_image}
ARG BASE_IMAGE_TAG={base_image_tag}
LABEL modelscope.base_image="vllm/vllm-openai-rocm:${BASE_IMAGE_TAG}"
# Build-time only (ARG does not persist into the image). Aliyun mirror can lag PyPI (~1h).
ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple
COPY docker/scripts/modelscope_env_init.sh /usr/local/bin/ms_env_init.sh
ARG CUR_TIME={cur_time}
RUN echo "CUR_TIME=${CUR_TIME}" && echo "BASE_IMAGE_TAG=${BASE_IMAGE_TAG}"
ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple
RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \
pip config set install.trusted-host mirrors.aliyun.com && \
cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b {modelscope_branch} --single-branch https://github.com/modelscope/modelscope.git && \
cd modelscope && pip install --no-cache-dir . -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
cd / && rm -fr /tmp/modelscope && pip cache purge
ENV VLLM_USE_MODELSCOPE=True
ENV LMDEPLOY_USE_MODELSCOPE=True
ENV MODELSCOPE_CACHE=/mnt/workspace/.cache/modelscope/hub
SHELL ["/bin/bash", "-c"]

View File

@@ -5,6 +5,8 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \
PIP_RETRIES=10 \ PIP_RETRIES=10 \
SOC_VERSION={soc_version} \ SOC_VERSION={soc_version} \
CANN_VERSION={cann_version} CANN_VERSION={cann_version}
# Build-time only (ARG does not persist into the image).
ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple
SHELL ["/bin/bash", "-c"] SHELL ["/bin/bash", "-c"]
@@ -36,12 +38,7 @@ RUN set -eux; \
esac esac
RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \ RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \
pip config set global.extra-index-url "https://pypi.org/simple" && \ pip config set install.trusted-host mirrors.aliyun.com
pip config set install.trusted-host mirrors.aliyun.com && \
ARCH=$(uname -m) && \
if [ "$ARCH" = "x86_64" ]; then \
pip config set global.extra-index-url "https://pypi.org/simple https://download.pytorch.org/whl/cpu/"; \
fi
{extra_content} {extra_content}
# ---------- Install vllm + vllm-ascend ---------- # ---------- Install vllm + vllm-ascend ----------
@@ -51,6 +48,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
git clone --depth 1 --branch {vllm_ascend_git_ref} https://github.com/vllm-project/vllm-ascend.git git clone --depth 1 --branch {vllm_ascend_git_ref} https://github.com/vllm-project/vllm-ascend.git
RUN ARCH=$(uname -m) && \ RUN ARCH=$(uname -m) && \
export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
source /usr/local/Ascend/nnal/atb/set_env.sh && \ source /usr/local/Ascend/nnal/atb/set_env.sh && \
# Install torch & torch_npu & torchvision # Install torch & torch_npu & torchvision
@@ -66,7 +64,8 @@ RUN git clone --depth 1 --branch {megatron_branch} https://github.com/NVIDIA/Meg
GIT_LFS_SKIP_SMUDGE=1 git clone --depth 1 -b {swift_branch} --single-branch https://github.com/modelscope/ms-swift.git /ms-swift GIT_LFS_SKIP_SMUDGE=1 git clone --depth 1 -b {swift_branch} --single-branch https://github.com/modelscope/ms-swift.git /ms-swift
# ---------- Install training-side repositories ---------- # ---------- Install training-side repositories ----------
RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \ if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \
cd /MindSpeed && pip install --no-cache-dir -e . && \ cd /MindSpeed && pip install --no-cache-dir -e . && \
pip install --no-cache-dir mcore-bridge -i https://pypi.org/simple/ -U && \ pip install --no-cache-dir mcore-bridge -i https://pypi.org/simple/ -U && \
@@ -75,7 +74,8 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
# ---------- Pin torch to the correct version + torch_npu ---------- # ---------- Pin torch to the correct version + torch_npu ----------
# x86: must force-install the CPU build from pytorch.org/whl/cpu # x86: must force-install the CPU build from pytorch.org/whl/cpu
# aarch64: PyPI only provides the CPU build, so install it directly from the Aliyun mirror # aarch64: PyPI only provides the CPU build, so install it directly from the Aliyun mirror
RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \ if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \
ARCH=$(uname -m) && \ ARCH=$(uname -m) && \
if [ "$ARCH" = "x86_64" ]; then \ if [ "$ARCH" = "x86_64" ]; then \
@@ -99,7 +99,8 @@ ENV PYTHONPATH=/Megatron-LM:${PYTHONPATH}
# install dependencies # install dependencies
COPY requirements /var/modelscope COPY requirements /var/modelscope
RUN pip uninstall ms-swift modelscope -y && pip install --no-cache-dir pip==23.* -U && \ RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
pip uninstall ms-swift modelscope -y && pip install --no-cache-dir pip==23.* -U && \
if [ "$INSTALL_MS_DEPS" = "True" ]; then \ if [ "$INSTALL_MS_DEPS" = "True" ]; then \
pip install --no-cache-dir omegaconf==2.0.6 && \ pip install --no-cache-dir omegaconf==2.0.6 && \
pip install 'editdistance==0.8.1' && \ pip install 'editdistance==0.8.1' && \
@@ -125,9 +126,11 @@ fi
ARG CUR_TIME={cur_time} ARG CUR_TIME={cur_time}
RUN echo $CUR_TIME RUN echo $CUR_TIME
RUN pip install --no-cache-dir --no-build-isolation OpenCC RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
pip install --no-cache-dir --no-build-isolation OpenCC
RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \ if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \
pip install --no-cache-dir -U funasr scikit-learn && \ pip install --no-cache-dir -U funasr scikit-learn && \
pip install --no-cache-dir -U qwen_vl_utils qwen_omni_utils librosa 'timm>=0.9.0' transformers accelerate peft trl safetensors && \ pip install --no-cache-dir -U qwen_vl_utils qwen_omni_utils librosa 'timm>=0.9.0' transformers accelerate peft trl safetensors && \
@@ -150,6 +153,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
# ---------- Install triton-ascend ---------- # ---------- Install triton-ascend ----------
RUN set -eux; \ RUN set -eux; \
export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}"; \
pip uninstall -y triton || true; \ pip uninstall -y triton || true; \
pip uninstall -y triton-ascend || true; \ pip uninstall -y triton-ascend || true; \
pip install --no-cache-dir --force-reinstall \ pip install --no-cache-dir --force-reinstall \

View File

@@ -3,6 +3,8 @@ FROM {base_image}
ARG DEBIAN_FRONTEND=noninteractive ARG DEBIAN_FRONTEND=noninteractive
ENV TZ=Asia/Shanghai ENV TZ=Asia/Shanghai
ENV arch=x86_64 ENV arch=x86_64
# Build-time only (ARG does not persist into the image). Aliyun mirror can lag PyPI (~1h).
ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple
COPY docker/scripts/modelscope_env_init.sh /usr/local/bin/ms_env_init.sh COPY docker/scripts/modelscope_env_init.sh /usr/local/bin/ms_env_init.sh
RUN apt-get update && \ RUN apt-get update && \
@@ -21,7 +23,9 @@ ARG IMAGE_TYPE={image_type}
# install dependencies # install dependencies
COPY requirements /var/modelscope COPY requirements /var/modelscope
RUN pip uninstall ms-swift modelscope -y && pip --no-cache-dir install pip==23.* -U && \ ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple
RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
pip uninstall ms-swift modelscope -y && pip --no-cache-dir install pip==23.* -U && \
if [ "$INSTALL_MS_DEPS" = "True" ]; then \ if [ "$INSTALL_MS_DEPS" = "True" ]; then \
pip --no-cache-dir install omegaconf==2.0.6 && \ pip --no-cache-dir install omegaconf==2.0.6 && \
pip install 'editdistance==0.8.1' && \ pip install 'editdistance==0.8.1' && \
@@ -34,7 +38,7 @@ if [ "$INSTALL_MS_DEPS" = "True" ]; then \
pip install --no-cache-dir 'scipy' && \ pip install --no-cache-dir 'scipy' && \
pip install --no-cache-dir funtextprocessing typeguard==2.13.3 scikit-learn -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ pip install --no-cache-dir funtextprocessing typeguard==2.13.3 scikit-learn -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
pip install --no-cache-dir 'decord>=0.6.0' mpi4py paint_ldm ipykernel fasttext -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ pip install --no-cache-dir 'decord>=0.6.0' mpi4py paint_ldm ipykernel fasttext -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
pip install --no-cache-dir ipywidgets && \ pip install --no-cache-dir ipywidgets jupyter_core nbconvert nbclient && \
pip install --no-cache-dir 'blobfile>=1.0.5' && \ pip install --no-cache-dir 'blobfile>=1.0.5' && \
pip uninstall MinDAEC -y && \ pip uninstall MinDAEC -y && \
pip install https://modelscope.oss-cn-beijing.aliyuncs.com/releases/dependencies/MinDAEC-0.0.2-py3-none-any.whl && \ pip install https://modelscope.oss-cn-beijing.aliyuncs.com/releases/dependencies/MinDAEC-0.0.2-py3-none-any.whl && \
@@ -47,7 +51,9 @@ fi
ARG CUR_TIME={cur_time} ARG CUR_TIME={cur_time}
RUN echo $CUR_TIME RUN echo $CUR_TIME
RUN bash /tmp/install.sh {version_args} && \ ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple
RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
bash /tmp/install.sh {version_args} && \
pip install --no-cache-dir -U funasr scikit-learn && \ pip install --no-cache-dir -U funasr scikit-learn && \
pip install --no-cache-dir -U qwen_vl_utils qwen_omni_utils librosa timm transformers accelerate peft trl safetensors && \ pip install --no-cache-dir -U qwen_vl_utils qwen_omni_utils librosa timm transformers accelerate peft trl safetensors && \
cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b {swift_branch} --single-branch https://github.com/modelscope/ms-swift.git && \ cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b {swift_branch} --single-branch https://github.com/modelscope/ms-swift.git && \
@@ -63,16 +69,17 @@ RUN bash /tmp/install.sh {version_args} && \
pip install --no-cache-dir transformers diffusers 'timm>=0.9.0' && pip cache purge; \ pip install --no-cache-dir transformers diffusers 'timm>=0.9.0' && pip cache purge; \
pip install --no-cache-dir omegaconf==2.3.0 && pip cache purge; \ pip install --no-cache-dir omegaconf==2.3.0 && pip cache purge; \
pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \ pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \
pip config set global.extra-index-url https://pypi.org/simple && \
pip config set install.trusted-host mirrors.aliyun.com && \ pip config set install.trusted-host mirrors.aliyun.com && \
cp /tmp/resources/ubuntu2204.aliyun /etc/apt/sources.list cp /tmp/resources/ubuntu2204.aliyun /etc/apt/sources.list
RUN if [ "$IMAGE_TYPE" = "gpu" ]; then \ ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple
RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \
if [ "$IMAGE_TYPE" = "gpu" ]; then \
pip install --no-cache-dir math_verify "gradio<5.33" "deepspeed<0.19" ray -U && \ pip install --no-cache-dir math_verify "gradio<5.33" "deepspeed<0.19" ray -U && \
pip install --no-cache-dir mcore-bridge -i https://pypi.org/simple/ -U && \ pip install --no-cache-dir mcore-bridge -i https://pypi.org/simple/ -U && \
pip install --no-cache-dir liger_kernel wandb swanlab nvitop pre-commit "transformers<5.9" "trl<1.0" "peft<0.20" huggingface-hub -U && \ pip install --no-cache-dir liger_kernel wandb swanlab nvitop pre-commit "transformers<5.9" "trl<1.0" "peft<0.20" huggingface-hub -U && \
pip install --no-cache-dir --no-build-isolation transformer_engine[pytorch]; \ pip install --no-cache-dir --no-build-isolation "transformer_engine[pytorch]==2.16.0"; \
cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone https://github.com/NVIDIA/apex && \ cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone https://github.com/NVIDIA/apex && \
cd apex && pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" ./ && \ cd apex && pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" ./ && \
cd / && rm -fr /tmp/apex && pip cache purge; \ cd / && rm -fr /tmp/apex && pip cache purge; \

View File

@@ -3,14 +3,27 @@ import os
import platform import platform
import re import re
import subprocess import subprocess
import urllib.error
import urllib.request
from copy import copy from copy import copy
from datetime import datetime from datetime import datetime
from typing import Any from typing import Any, List, Optional
import json
docker_registry = os.environ['DOCKER_REGISTRY'] docker_registry = os.environ['DOCKER_REGISTRY']
assert docker_registry, 'You must pass a valid DOCKER_REGISTRY' assert docker_registry, 'You must pass a valid DOCKER_REGISTRY'
timestamp = datetime.now() timestamp = datetime.now()
formatted_time = timestamp.strftime('%Y%m%d%H%M%S') formatted_time = timestamp.strftime('%Y%m%d%H%M%S')
VLLM_ROCM_REPO = 'vllm/vllm-openai-rocm'
_FLOATING_ROCM_TAGS = frozenset({
'latest',
'latest-base',
'nightly',
'base-nightly',
})
_VERSION_TAG_PATTERN = re.compile(r'^v\d+(?:\.\d+)*$')
_NIGHTLY_HASH_PATTERN = re.compile(r'^(?:base-)?nightly-[0-9a-f]{7,40}$')
class Builder: class Builder:
@@ -359,7 +372,8 @@ class StableGPUImageBuilder(Builder):
extra_content = extra_content.replace('{python_version}', extra_content = extra_content.replace('{python_version}',
self.args.python_version) self.args.python_version)
extra_content += """ extra_content += """
RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy RUN export PIP_EXTRA_INDEX_URL=https://pypi.org/simple && \
pip install --no-cache-dir -U icecream soundfile pybind11 py-spy
""" """
version_args = ( version_args = (
f'{self.args.torch_version} {self.args.torchvision_version} {self.args.torchaudio_version} ' f'{self.args.torch_version} {self.args.torchvision_version} {self.args.torchaudio_version} '
@@ -420,7 +434,8 @@ class LatestGPUImageBuilder(StableGPUImageBuilder):
extra_content = extra_content.replace('{python_version}', extra_content = extra_content.replace('{python_version}',
self.args.python_version) self.args.python_version)
extra_content += """ extra_content += """
RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy RUN export PIP_EXTRA_INDEX_URL=https://pypi.org/simple && \
pip install --no-cache-dir -U icecream soundfile pybind11 py-spy
""" """
version_args = ( version_args = (
f'{self.args.torch_version} {self.args.torchvision_version} {self.args.torchaudio_version} ' f'{self.args.torch_version} {self.args.torchvision_version} {self.args.torchaudio_version} '
@@ -467,6 +482,370 @@ RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy
return self.run_cmd('docker', 'push', image_tag2) return self.run_cmd('docker', 'push', image_tag2)
class AmdImageBuilder(Builder):
"""Build ModelScope image on top of vllm/vllm-openai-rocm."""
@staticmethod
def _is_specific_release_tag(tag: str) -> bool:
tag = tag.strip()
if not tag or tag.lower() in _FLOATING_ROCM_TAGS:
return False
if tag.endswith('-base'):
return False
if _NIGHTLY_HASH_PATTERN.fullmatch(tag):
return False
return bool(_VERSION_TAG_PATTERN.fullmatch(tag))
@staticmethod
def _image_digest(tag_info: dict) -> Optional[str]:
digest = tag_info.get('digest')
if digest:
return digest
for image in tag_info.get('images') or []:
digest = image.get('digest')
if digest:
return digest
return None
@classmethod
def _fetch_rocm_tags(cls, page_size: int = 100) -> List[dict]:
tags: List[dict] = []
url = (f'https://hub.docker.com/v2/repositories/{VLLM_ROCM_REPO}/tags'
f'?page_size={page_size}&ordering=-last_updated')
while url:
req = urllib.request.Request(
url, headers={'User-Agent': 'modelscope-docker-builder'})
try:
with urllib.request.urlopen(req, timeout=60) as resp:
payload = json.load(resp)
except (urllib.error.URLError, json.JSONDecodeError) as exc:
raise RuntimeError(
f'Failed to query Docker Hub tags for {VLLM_ROCM_REPO}: '
f'{exc}') from exc
tags.extend(payload.get('results') or [])
url = payload.get('next')
# Only scan the first few pages; release tags are near the top.
if len(tags) >= 300:
break
if not tags:
raise RuntimeError(
f'No tags returned from Docker Hub for {VLLM_ROCM_REPO}')
return tags
@classmethod
def resolve_latest_rocm_tag(cls) -> str:
"""Resolve the newest concrete release tag for vllm-openai-rocm.
Preference order:
1. Semver tag (vX.Y.Z) that shares digest with floating ``latest``
2. Newest semver tag by Docker Hub ``last_updated``
"""
tags = cls._fetch_rocm_tags()
by_name = {item['name']: item for item in tags if item.get('name')}
release_tags = [
item for item in tags
if cls._is_specific_release_tag(item.get('name', ''))
]
latest_info = by_name.get('latest')
latest_digest = cls._image_digest(latest_info) if latest_info else None
if latest_digest:
matched = [
item for item in release_tags
if cls._image_digest(item) == latest_digest
]
if matched:
# Prefer the first match in last_updated order from API.
chosen = matched[0]['name']
print(
f'Resolved {VLLM_ROCM_REPO} latest digest to release tag: '
f'{chosen}')
return chosen
if not release_tags:
raise RuntimeError(
f'No concrete release tags found for {VLLM_ROCM_REPO}')
chosen = release_tags[0]['name']
print(f'Resolved newest {VLLM_ROCM_REPO} release tag: {chosen}')
return chosen
def init_args(self, args: Any) -> Any:
# Auto-discover from Docker Hub unless an explicit override is given.
override = getattr(args, 'base_image_tag', None)
if override and str(override).strip() and str(
override).strip().lower() not in {'auto', 'latest'}:
args.base_image_tag = str(override).strip()
if not self._is_specific_release_tag(args.base_image_tag):
raise ValueError(
'base_image_tag override must be a concrete release tag '
f'(e.g. v0.25.1), got: {args.base_image_tag}')
print(f'Using override AMD ROCm base image tag: '
f'{args.base_image_tag}')
else:
args.base_image_tag = self.resolve_latest_rocm_tag()
if not args.base_image:
args.base_image = f'{VLLM_ROCM_REPO}:{args.base_image_tag}'
if not args.cuda_version:
args.cuda_version = '0.0.0'
return args
@staticmethod
def _sanitize_tag(tag: str) -> str:
return re.sub(r'[^A-Za-z0-9._-]+', '-', tag)
@staticmethod
def _normalize_version(version: str) -> str:
version = version.strip().lstrip('vV')
version = version.split('+')[0].split(' ')[0]
return re.sub(r'[^0-9A-Za-z._-]+', '', version)
@staticmethod
def _python_tag_from_version(version: str) -> str:
parts = version.strip().split('.')
if len(parts) >= 2 and parts[0].isdigit() and parts[1].isdigit():
return f'py{parts[0]}{parts[1]}'
return f'py{re.sub(r"[^0-9]", "", version)}'
@classmethod
def _run_capture(cls, *cmd: str) -> subprocess.CompletedProcess:
return subprocess.run(
list(cmd), capture_output=True, text=True, check=False)
@classmethod
def _probe_via_entrypoint(cls, base_image: str) -> dict:
"""Read versions with docker run --entrypoint (no GPU required)."""
# Keep this script compact: it runs inside the base image via python -c.
script = (
'import json,os,pathlib,subprocess,sys\n'
'info={"python":"%d.%d.%d"%sys.version_info[:3]}\n'
'try:\n'
' import torch\n'
' info["torch"]=torch.__version__\n'
' hip=getattr(torch.version,"hip",None)\n'
' if hip: info["torch_hip"]=hip\n'
'except Exception as e:\n'
' info["torch_error"]=str(e)\n'
'for p in ("/opt/rocm/.info/version","/opt/rocm/.info/version-dev"):\n'
' f=pathlib.Path(p)\n'
' if f.is_file():\n'
' info["rocm_file"]=f.read_text().strip().splitlines()[0]\n'
' break\n'
'for k in ("ROCM_VERSION","HIP_VERSION","TORCH_VERSION"):\n'
' if os.environ.get(k): info[k.lower()]=os.environ[k]\n'
'def _dpkg_ver(*names):\n'
' for n in names:\n'
' try:\n'
' r=subprocess.run(["dpkg-query","-W","-f=${Version}",n],'
'capture_output=True,text=True)\n'
' if r.returncode==0 and r.stdout.strip():\n'
' return r.stdout.strip()\n'
' except Exception:\n'
' pass\n'
' return None\n'
'def _dpkg_scan(prefixes):\n'
' try:\n'
' r=subprocess.run(["dpkg-query","-W","-f=${Package}\\t${Version}\\n"],'
'capture_output=True,text=True)\n'
' except Exception:\n'
' return {}\n'
' found={}\n'
' for line in (r.stdout or "").splitlines():\n'
' if "\\t" not in line: continue\n'
' pkg,ver=line.split("\\t",1)\n'
' for pref in prefixes:\n'
' if pkg==pref or pkg.startswith(pref+"-"):\n'
' found.setdefault(pref,ver)\n'
' return found\n'
'pkgs=_dpkg_scan(("rccl","miopen"))\n'
'info["system.library.rccl"]=_dpkg_ver("rccl") or pkgs.get("rccl")\n'
'info["system.library.miopen"]=('
'_dpkg_ver("miopen-hip","miopen") or pkgs.get("miopen"))\n'
'print(json.dumps(info))\n')
for py in ('python3', 'python'):
result = cls._run_capture('docker', 'run', '--rm', '--network',
'none', '--entrypoint', py, base_image,
'-c', script)
if result.returncode == 0 and result.stdout.strip():
try:
return json.loads(result.stdout.strip().splitlines()[-1])
except json.JSONDecodeError:
continue
return {}
@classmethod
def _probe_via_history(cls, base_image: str) -> dict:
"""Parse build ARGs from docker history (no container start)."""
result = cls._run_capture('docker', 'history', '--no-trunc',
'--format', '{{.CreatedBy}}', base_image)
if result.returncode != 0:
return {}
text = result.stdout
info = {}
for key, pattern in (
('rocm', r'ROCM_VERSION=([0-9]+(?:\.[0-9]+)*)'),
('python', r'PYTHON_VERSION=([0-9]+(?:\.[0-9]+)*)'),
('ubuntu',
r'org\.opencontainers\.image\.version=([0-9]+(?:\.[0-9]+)*)'),
):
matches = re.findall(pattern, text)
if matches:
# docker history lists newest layers first.
info[key] = matches[0]
return info
@classmethod
def _probe_via_create_cp(cls, base_image: str) -> dict:
"""Copy version files out of a created (not started) container."""
import tempfile
create = cls._run_capture('docker', 'create', base_image)
if create.returncode != 0:
return {}
cid = create.stdout.strip()
info = {}
try:
with tempfile.TemporaryDirectory() as tmp:
dest = os.path.join(tmp, 'version')
for src in ('/opt/rocm/.info/version',
'/opt/rocm/.info/version-dev'):
result = cls._run_capture('docker', 'cp', f'{cid}:{src}',
dest)
if result.returncode == 0 and os.path.isfile(dest):
with open(dest, 'r', encoding='utf-8') as f:
line = f.read().strip().splitlines()
if line:
info['rocm_file'] = line[0].strip()
break
finally:
cls._run_capture('docker', 'rm', '-f', cid)
return info
@classmethod
def probe_base_image_versions(cls, base_image: str) -> dict:
"""Discover rocm/python/torch without needing AMD GPU.
Methods (in order):
1. docker run --entrypoint python -c ... (CPU-only, no --device)
2. docker history --no-trunc parse ROCM_VERSION/PYTHON_VERSION
3. docker create + docker cp /opt/rocm/.info/version
"""
probed = {}
entry = cls._probe_via_entrypoint(base_image)
history = cls._probe_via_history(base_image)
copied = cls._probe_via_create_cp(base_image)
probed.update(history)
probed.update(copied)
probed.update(entry)
rocm = (
probed.get('rocm_file') or probed.get('rocm_version')
or probed.get('rocm') or probed.get('torch_hip')
or probed.get('hip_version'))
python_ver = probed.get('python')
torch_ver = probed.get('torch') or probed.get('torch_version')
ubuntu_ver = probed.get('ubuntu')
# Keep dpkg package versions as-is (may contain '~', e.g. 2.27.7.70201-81~22.04).
rccl_ver = probed.get('system.library.rccl')
miopen_ver = probed.get('system.library.miopen')
versions = {
'rocm': cls._normalize_version(rocm) if rocm else None,
'python':
cls._normalize_version(python_ver) if python_ver else None,
'torch': cls._normalize_version(torch_ver) if torch_ver else None,
'ubuntu':
cls._normalize_version(ubuntu_ver) if ubuntu_ver else None,
'system.library.rccl': rccl_ver or None,
'system.library.miopen': miopen_ver or None,
}
print('Probed AMD base image versions:')
for key, value in versions.items():
print(f' {key}: {value or "unknown"}')
return versions
def generate_dockerfile(self) -> str:
with open('docker/Dockerfile.amd', 'r') as f:
content = f.read()
content = content.replace('{base_image}', self.args.base_image)
content = content.replace('{base_image_tag}', self.args.base_image_tag)
content = content.replace('{modelscope_branch}',
self.args.modelscope_branch)
content = content.replace('{cur_time}', formatted_time)
return content
def image(self) -> str:
ubuntu = getattr(self.args, 'amd_ubuntu_version',
None) or self.args.ubuntu_version
rocm = getattr(self.args, 'amd_rocm_version', None)
py_tag = getattr(self.args, 'amd_python_tag', None) or getattr(
self.args, 'python_tag', None)
torch = getattr(self.args, 'amd_torch_version', None)
if not (rocm and py_tag and torch):
raise RuntimeError(
'AMD image tag requires probed rocm/python/torch versions. '
f'Got rocm={rocm}, python={py_tag}, torch={torch}')
return (f'{docker_registry}:ubuntu{ubuntu}-rocm{rocm}-{py_tag}-'
f'torch{torch}-{self.args.modelscope_version}-test')
def _log_base_image_info(self) -> int:
base_image = self.args.base_image
print('=' * 60)
print(f'AMD ROCm base image: {base_image}')
print(f'AMD ROCm base image tag: {self.args.base_image_tag}')
print('=' * 60)
ret = self.run_cmd('docker', 'pull', base_image)
if ret != 0:
return ret
result = self._run_capture(
'docker', 'image', 'inspect', base_image,
'--format={{.Id}} {{if index .RepoDigests 0}}'
'{{index .RepoDigests 0}}{{else}}local-only{{end}}')
if result.returncode == 0:
print(f'AMD base image resolved: {result.stdout.strip()}')
else:
print(f'AMD base image inspect warning: {result.stderr.strip()}')
versions = self.probe_base_image_versions(base_image)
if not versions.get('rocm') or not versions.get(
'python') or not versions.get('torch'):
print('ERROR: failed to probe rocm/python/torch from base image')
return 1
self.args.amd_rocm_version = versions['rocm']
self.args.amd_torch_version = versions['torch']
self.args.amd_python_tag = self._python_tag_from_version(
versions['python'])
if versions.get('ubuntu'):
self.args.amd_ubuntu_version = versions['ubuntu']
else:
self.args.amd_ubuntu_version = self.args.ubuntu_version
print(f'AMD output image tag will be: {self.image()}')
print('=' * 60)
return 0
def build(self) -> int:
ret = self._log_base_image_info()
if ret != 0:
return ret
return self.run_cmd('docker', 'build', '-t', self.image(), '-f',
'Dockerfile', '.')
def push(self):
image_name = self.image()
ret = self.run_cmd('docker', 'push', image_name)
if ret != 0:
return ret
ubuntu = self.args.amd_ubuntu_version
rocm = self.args.amd_rocm_version
py_tag = self.args.amd_python_tag
torch = self.args.amd_torch_version
image_tag2 = (f'{docker_registry}:ubuntu{ubuntu}-rocm{rocm}-{py_tag}-'
f'torch{torch}-{self.args.modelscope_version}-'
f'{formatted_time}-test')
ret = self.run_cmd('docker', 'tag', image_name, image_tag2)
if ret != 0:
return ret
print(f'AMD image timestamp tag: {image_tag2}')
return self.run_cmd('docker', 'push', image_tag2)
class AscendImageBuilder(StableGPUImageBuilder): class AscendImageBuilder(StableGPUImageBuilder):
_DEFAULT_TORCH_VERSION = '2.9.0' _DEFAULT_TORCH_VERSION = '2.9.0'
@@ -638,7 +1017,8 @@ class AscendImageBuilder(StableGPUImageBuilder):
def generate_dockerfile(self) -> str: def generate_dockerfile(self) -> str:
extra_content = """ extra_content = """
RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy RUN export PIP_EXTRA_INDEX_URL=https://pypi.org/simple && \
pip install --no-cache-dir -U icecream soundfile pybind11 py-spy
""" """
with open('docker/Dockerfile.ascend', 'r') as f: with open('docker/Dockerfile.ascend', 'r') as f:
content = f.read() content = f.read()
@@ -708,6 +1088,12 @@ parser.add_argument('--megatron_branch', type=str, default='v0.15.3')
parser.add_argument('--mindspeed_branch', type=str, default='core_r0.15.3') parser.add_argument('--mindspeed_branch', type=str, default='core_r0.15.3')
parser.add_argument('--soc_version', type=str, default='ascend910_9391') parser.add_argument('--soc_version', type=str, default='ascend910_9391')
parser.add_argument('--arch', type=str, choices=['x86', 'arm'], default=None) parser.add_argument('--arch', type=str, choices=['x86', 'arm'], default=None)
parser.add_argument(
'--base_image_tag',
type=str,
default=None,
help='Optional AMD ROCm override tag. Default: auto-resolve newest '
'concrete vllm/vllm-openai-rocm release tag from Docker Hub.')
parser.add_argument('--dry_run', type=int, default=0) parser.add_argument('--dry_run', type=int, default=0)
args = parser.parse_args() args = parser.parse_args()
@@ -719,6 +1105,8 @@ elif args.image_type.lower() == 'stable':
builder_cls = [StableCPUImageBuilder, StableGPUImageBuilder] builder_cls = [StableCPUImageBuilder, StableGPUImageBuilder]
elif args.image_type.lower() == 'ascend': elif args.image_type.lower() == 'ascend':
builder_cls = [AscendImageBuilder] builder_cls = [AscendImageBuilder]
elif args.image_type.lower() == 'amd':
builder_cls = [AmdImageBuilder]
elif args.image_type.lower() == 'latest': elif args.image_type.lower() == 'latest':
builder_cls = [LatestGPUImageBuilder] builder_cls = [LatestGPUImageBuilder]
else: else:

View File

@@ -47,5 +47,4 @@ else
fi fi
pip config set global.index-url https://mirrors.cloud.aliyuncs.com/pypi/simple pip config set global.index-url https://mirrors.cloud.aliyuncs.com/pypi/simple
pip config set global.extra-index-url https://pypi.org/simple
pip config set install.trusted-host mirrors.cloud.aliyuncs.com pip config set install.trusted-host mirrors.cloud.aliyuncs.com

View File

@@ -18,7 +18,8 @@ from typing import Dict, List, Optional, Type
import requests import requests
# --- Hub file downloads (delegated) --- # --- Hub file downloads (delegated) ---
from modelscope_hub.compat import dataset_file_download # noqa: E402,F401 from modelscope_hub.compat.file_download import \
dataset_file_download as _compat_dataset_file_download
from modelscope_hub.compat.file_download import \ from modelscope_hub.compat.file_download import \
model_file_download as _compat_model_file_download model_file_download as _compat_model_file_download
from requests.adapters import Retry from requests.adapters import Retry
@@ -31,7 +32,7 @@ from modelscope.hub.constants import (API_FILE_DOWNLOAD_CHUNK_SIZE,
from modelscope.utils.logger import get_logger from modelscope.utils.logger import get_logger
from .callback import ProgressCallback, TqdmCallback from .callback import ProgressCallback, TqdmCallback
from .errors import FileDownloadError from .errors import FileDownloadError
from .utils.utils import get_endpoint from .utils.utils import find_reusable_legacy_repo_dir, get_endpoint
logger = get_logger() logger = get_logger()
@@ -77,6 +78,9 @@ def model_file_download(
revision = detail.get('Revision') revision = detail.get('Revision')
except Exception: except Exception:
pass pass
if local_dir is None:
local_dir = find_reusable_legacy_repo_dir(
model_id, repo_type='model', cache_dir=cache_dir)
return _compat_model_file_download( return _compat_model_file_download(
model_id, model_id,
file_path, file_path,
@@ -91,6 +95,37 @@ def model_file_download(
) )
def dataset_file_download(
dataset_id: str,
file_path: str,
*,
cache_dir: str = None,
local_dir: str = None,
revision: str = None,
cookies: dict = None,
token: str = None,
endpoint: str = None,
local_files_only: bool = False,
user_agent=None,
) -> str:
"""Download a single dataset file, reusing flat/hub legacy caches when present."""
if local_dir is None:
local_dir = find_reusable_legacy_repo_dir(
dataset_id, repo_type='dataset', cache_dir=cache_dir)
return _compat_dataset_file_download(
dataset_id,
file_path,
cache_dir=cache_dir,
local_dir=local_dir,
revision=revision,
cookies=cookies,
token=token,
endpoint=endpoint,
local_files_only=local_files_only,
user_agent=user_agent,
)
# --- Direct HTTP downloads (retained - non-Hub API) --- # --- Direct HTTP downloads (retained - non-Hub API) ---

View File

@@ -4,16 +4,58 @@ Delegates to ``modelscope_hub.compat`` while keeping ``revision``, ``cache_dir``
and friends accessible as positional arguments for backward compatibility. and friends accessible as positional arguments for backward compatibility.
""" """
from __future__ import annotations from __future__ import annotations
import threading
from pathlib import Path from pathlib import Path
from typing import Dict, List, Optional, Union from typing import TYPE_CHECKING, Dict, List, Optional, Type, Union
from modelscope_hub.compat.snapshot_download import \ from modelscope_hub.compat.snapshot_download import \
dataset_snapshot_download as _compat_dataset_snapshot_download dataset_snapshot_download as _compat_dataset_snapshot_download
from modelscope_hub.compat.snapshot_download import \ from modelscope_hub.compat.snapshot_download import \
snapshot_download as _compat_snapshot_download snapshot_download as _compat_snapshot_download
from modelscope.hub.utils.utils import find_reusable_legacy_repo_dir
from modelscope.utils.logger import get_logger
if TYPE_CHECKING:
from .callback import ProgressCallback
logger = get_logger()
__all__ = ['snapshot_download', 'dataset_snapshot_download'] __all__ = ['snapshot_download', 'dataset_snapshot_download']
# Capability probe: pre-1.38 cache auto-detection lives in modelscope-hub
# (DownloadManager._find_legacy_repo_dir, added in modelscope-hub>=0.1.7).
# Warn once if the installed hub predates it, so an existing legacy cache is
# not silently ignored and re-downloaded into the new layout.
_legacy_cache_capability: Optional[bool] = None
_legacy_cache_lock = threading.Lock()
def _warn_if_legacy_cache_detection_unavailable() -> None:
"""Warn once when the installed modelscope-hub cannot auto-detect a
pre-1.38 (legacy) cache layout, so downloads don't silently skip an
existing local cache and re-fetch into the new layout.
"""
global _legacy_cache_capability
if _legacy_cache_capability is not None:
return
with _legacy_cache_lock:
if _legacy_cache_capability is not None:
return
try:
from modelscope_hub._download import DownloadManager
_legacy_cache_capability = hasattr(DownloadManager,
'_find_legacy_repo_dir')
except Exception:
_legacy_cache_capability = False
if not _legacy_cache_capability:
logger.warning(
'The installed modelscope-hub lacks legacy cache '
'auto-detection (added in modelscope-hub>=0.1.7). An existing '
'pre-1.38 cache will not be reused; files will be downloaded '
'into the new cache layout. Upgrade with: '
"pip install -U 'modelscope-hub>=0.1.8'.")
def snapshot_download( def snapshot_download(
model_id: Optional[str] = None, model_id: Optional[str] = None,
@@ -30,6 +72,7 @@ def snapshot_download(
max_workers: Optional[int] = None, max_workers: Optional[int] = None,
repo_id: Optional[str] = None, repo_id: Optional[str] = None,
repo_type: Optional[str] = None, repo_type: Optional[str] = None,
progress_callbacks: Optional[List[Type[ProgressCallback]]] = None,
token: Optional[str] = None, token: Optional[str] = None,
endpoint: Optional[str] = None, endpoint: Optional[str] = None,
) -> str: ) -> str:
@@ -37,11 +80,20 @@ def snapshot_download(
Preserves the legacy positional-argument signature for backward Preserves the legacy positional-argument signature for backward
compatibility while delegating to ``modelscope_hub.compat``. compatibility while delegating to ``modelscope_hub.compat``.
``progress_callbacks`` is a list of :class:`ProgressCallback` subclasses
(not instances), each instantiated per file to report download progress.
""" """
_warn_if_legacy_cache_detection_unavailable()
effective_id = repo_id or model_id
effective_type = repo_type or 'model'
cache_dir_str = str(cache_dir) if cache_dir is not None else None
if local_dir is None and effective_id is not None:
local_dir = find_reusable_legacy_repo_dir(
effective_id, repo_type=effective_type, cache_dir=cache_dir_str)
return _compat_snapshot_download( return _compat_snapshot_download(
model_id=model_id, model_id=model_id,
revision=revision, revision=revision,
cache_dir=str(cache_dir) if cache_dir is not None else None, cache_dir=cache_dir_str,
local_dir=local_dir, local_dir=local_dir,
allow_file_pattern=allow_file_pattern, allow_file_pattern=allow_file_pattern,
ignore_file_pattern=ignore_file_pattern, ignore_file_pattern=ignore_file_pattern,
@@ -56,6 +108,7 @@ def snapshot_download(
local_files_only=bool(local_files_only) local_files_only=bool(local_files_only)
if local_files_only is not None else False, if local_files_only is not None else False,
user_agent=user_agent, user_agent=user_agent,
progress_callbacks=progress_callbacks,
) )
@@ -75,11 +128,16 @@ def dataset_snapshot_download(
endpoint: Optional[str] = None, endpoint: Optional[str] = None,
) -> str: ) -> str:
"""Download a dataset repo snapshot (legacy positional-arg signature).""" """Download a dataset repo snapshot (legacy positional-arg signature)."""
_warn_if_legacy_cache_detection_unavailable()
effective_id = dataset_id or repo_id effective_id = dataset_id or repo_id
cache_dir_str = str(cache_dir) if cache_dir is not None else None
if local_dir is None and effective_id is not None:
local_dir = find_reusable_legacy_repo_dir(
effective_id, repo_type='dataset', cache_dir=cache_dir_str)
return _compat_dataset_snapshot_download( return _compat_dataset_snapshot_download(
dataset_id=effective_id, dataset_id=effective_id,
revision=revision, revision=revision,
cache_dir=str(cache_dir) if cache_dir is not None else None, cache_dir=cache_dir_str,
local_dir=local_dir, local_dir=local_dir,
allow_file_pattern=allow_file_pattern, allow_file_pattern=allow_file_pattern,
ignore_file_pattern=ignore_file_pattern, ignore_file_pattern=ignore_file_pattern,

View File

@@ -194,6 +194,78 @@ def get_cache_dir(model_id: Optional[str] = None):
base_path, model_id + '/') base_path, model_id + '/')
def _modelscope_hub_cache_root() -> Path:
"""Cache root used by ``modelscope_hub`` downloads (not SDK ``.../hub``)."""
env = os.environ.get('MODELSCOPE_CACHE')
if env:
return Path(env).expanduser()
return Path.home() / '.cache' / 'modelscope'
def find_reusable_legacy_repo_dir(
repo_id: str,
repo_type: str = 'model',
cache_dir: Optional[Union[str, Path]] = None,
) -> Optional[str]:
"""Find old on-disk cache layouts that ``modelscope_hub`` download misses.
``modelscope_hub`` reuses ``{cache}/{type}s/{owner}/{safe_name}/`` (dots in
``name`` replaced by ``___``) and writes to
``{cache}/{type}s/{owner}--{name}/snapshots/{rev}/``. Older SDKs also
stored repos at:
- ``{cache}/{owner}/{name}/`` (flat, when ``MODELSCOPE_CACHE`` was set)
- ``{cache}/hub/{owner}/{name}/`` (pre-``models/`` restructuring)
- ``{cache}/{type}s/{owner}/{name}/`` (unsafed name; hub only checks
``safe_name``)
Returns a non-empty legacy path only when the layouts hub already handles
are absent, so callers can pass it as ``local_dir`` and avoid re-download.
"""
if not repo_id or '/' not in repo_id:
return None
base = Path(cache_dir).expanduser() if cache_dir is not None else \
_modelscope_hub_cache_root()
segment = f'{repo_type}s' if not repo_type.endswith('s') else repo_type
owner, name = repo_id.split('/', 1)
safe_name = name.replace('.', '___')
safe_id = repo_id.replace('/', '--')
# Layouts already handled by modelscope_hub — do not override.
hub_known = [
base / segment / safe_id,
base / segment / owner / safe_name,
]
for path in hub_known:
if _non_empty_dir(path):
return None
# Layouts hub download does not probe today.
legacy_candidates = [
base / owner / name,
base / owner / safe_name,
base / 'hub' / owner / name,
base / 'hub' / owner / safe_name,
base / segment / owner / name,
]
for path in legacy_candidates:
if _non_empty_dir(path):
logger.info('Found legacy cache at %s for %s, reusing.', path,
repo_id)
return str(path)
return None
def _non_empty_dir(path: Path) -> bool:
if not path.is_dir():
return False
try:
return any(path.iterdir())
except OSError:
return False
def get_release_datetime(): def get_release_datetime():
if MODELSCOPE_SDK_DEBUG in os.environ: if MODELSCOPE_SDK_DEBUG in os.environ:
rt = int(round(datetime.now().timestamp())) rt = int(round(datetime.now().timestamp()))

View File

@@ -147,12 +147,13 @@ def check_model_from_owner_group(model_dir: str,
if group in owner_group: if group in owner_group:
return True return True
# Also check cache path pattern: {cache_root}/{owner}--{model_name}/snapshots/{revision} # Also check cache path pattern: {cache_root}/{owner}--{model_name}/snapshots/{revision}
# Require exactly "{owner}--{name}" format (2 segments split by --) # Require exactly "{owner}--{name}" with both segments non-empty
# to prevent spoofing via accounts like "iic--hacked" which would # to prevent spoofing via accounts like "iic--hacked" (paths like
# produce paths like "iic--hacked--evil" and bypass the check. # "iic--hacked--evil") or empty names like "iic--".
grandparent = os.path.basename(os.path.dirname(parent_dir)) grandparent = os.path.basename(os.path.dirname(parent_dir))
if '--' in grandparent: if '--' in grandparent:
parts = grandparent.split('--') parts = grandparent.split('--')
if len(parts) == 2 and parts[0] in owner_group: # Both owner and name must be non-empty; reject "iic--" / "--name".
if len(parts) == 2 and all(parts) and parts[0] in owner_group:
return True return True
return False return False

View File

@@ -155,24 +155,82 @@ def _decide_allow_file_pattern(module_name, cls=None):
return extra_allow_file_pattern return extra_allow_file_pattern
def _ms_revision(revision):
"""Translate an HF revision string into one ModelScope accepts."""
return 'master' if revision in (None, 'main') else revision
def _ms_download_kwargs_from_hf(kwargs, revision=None):
"""Map transformers download kwargs onto ``snapshot_download`` arguments.
Forwards ``local_files_only``, ``cache_dir``, string ``token``, and an
optional revision (normalized via ``_ms_revision``).
"""
download_kwargs = {
'local_files_only': kwargs.get('local_files_only', False),
}
cache_dir = kwargs.get('cache_dir')
if cache_dir is not None:
download_kwargs['cache_dir'] = cache_dir
token = kwargs.get('token')
if isinstance(token, str):
download_kwargs['token'] = token
if revision is not None:
download_kwargs['revision'] = _ms_revision(revision)
return download_kwargs
def _get_class_from_dynamic_module(class_reference, *args, **kwargs): def _get_class_from_dynamic_module(class_reference, *args, **kwargs):
"""Wrapper that redirects dynamic-module downloads to ModelScope. """Wrapper that redirects dynamic-module downloads to ModelScope.
When a config's ``auto_map`` references another repo, transformers calls When a config's ``auto_map`` references another repo, transformers calls
``get_class_from_dynamic_module`` to fetch it. This wrapper ensures that ``get_class_from_dynamic_module`` to fetch it. This wrapper ensures that
fetch goes through ModelScope instead of HuggingFace. fetch goes through ModelScope instead of HuggingFace.
Cross-repo ``auto_map`` entries use ``repo_id--module.Class``. After
``snapshot_download``, the local cache path may itself contain ``--``
(modelscope_hub 0.1.x layout: ``models/{owner}--{name}/snapshots/...``).
Re-joining that path with ``--`` would make transformers'
``class_reference.split("--")`` raise ``ValueError``. Instead, pass the
local directory as ``pretrained_model_name_or_path`` and the bare
``module.Class`` as ``class_reference`` so transformers takes the
``os.path.isdir`` branch.
""" """
from transformers.dynamic_module_utils import origin_get_class_from_dynamic_module from transformers.dynamic_module_utils import origin_get_class_from_dynamic_module
if 'pretrained_model_name_or_path' in inspect.signature( has_pretrained_arg = (
origin_get_class_from_dynamic_module).parameters: 'pretrained_model_name_or_path'
pretrained_model_name_or_path = args[0] in inspect.signature(origin_get_class_from_dynamic_module).parameters)
if not os.path.exists(pretrained_model_name_or_path): # Resolve pretrained_model_name_or_path from kwargs or positional args.
from modelscope import snapshot_download # ``args`` is a tuple; never mutate it in place.
args[0] = snapshot_download(pretrained_model_name_or_path) pretrained_in_kwargs = False
pretrained_model_name_or_path = None
if has_pretrained_arg:
if 'pretrained_model_name_or_path' in kwargs:
pretrained_model_name_or_path = kwargs[
'pretrained_model_name_or_path']
pretrained_in_kwargs = True
elif args:
pretrained_model_name_or_path = args[0]
if (pretrained_model_name_or_path is not None
and not os.path.exists(pretrained_model_name_or_path)):
from modelscope import snapshot_download
# Model weights/config: use ``revision`` (not ``code_revision``).
downloaded_path = snapshot_download(
pretrained_model_name_or_path,
**_ms_download_kwargs_from_hf(
kwargs, revision=kwargs.get('revision')))
if pretrained_in_kwargs:
kwargs['pretrained_model_name_or_path'] = downloaded_path
else:
args = (downloaded_path, ) + args[1:]
pretrained_model_name_or_path = downloaded_path
if '--' in class_reference: if '--' in class_reference:
repo_id, class_reference = class_reference.split('--') # Only the first ``--`` is the auto_map delimiter (repo vs module).
repo_id, class_reference = class_reference.split('--', 1)
if not os.path.exists(repo_id): if not os.path.exists(repo_id):
download_kwargs = {} # Cross-repo code: transformers uses ``code_revision`` for this repo.
download_kwargs = _ms_download_kwargs_from_hf(
kwargs, revision=kwargs.get('code_revision'))
extra_allow_file_pattern = _decide_allow_file_pattern( extra_allow_file_pattern = _decide_allow_file_pattern(
class_reference) class_reference)
if extra_allow_file_pattern is not None: if extra_allow_file_pattern is not None:
@@ -182,7 +240,18 @@ def _get_class_from_dynamic_module(class_reference, *args, **kwargs):
download_kwargs['ignore_file_pattern'] = ignore_file_pattern download_kwargs['ignore_file_pattern'] = ignore_file_pattern
from modelscope import snapshot_download from modelscope import snapshot_download
repo_id = snapshot_download(repo_id, **download_kwargs) repo_id = snapshot_download(repo_id, **download_kwargs)
class_reference = repo_id + '--' + class_reference if has_pretrained_arg:
# Local path + bare class name; do not rejoin with ``--``.
# Keep kwargs/positional form consistent with the original call.
if pretrained_in_kwargs:
kwargs['pretrained_model_name_or_path'] = repo_id
else:
args = (repo_id, ) + args[1:]
else:
# Legacy transformers without pretrained_model_name_or_path.
# Unsafe if repo_id (local cache) contains '--'; modern
# transformers always take the branch above.
class_reference = repo_id + '--' + class_reference
return origin_get_class_from_dynamic_module(class_reference, *args, return origin_get_class_from_dynamic_module(class_reference, *args,
**kwargs) **kwargs)
@@ -248,18 +317,16 @@ def _patch_pretrained_class(all_imported_modules, wrap=False):
if subfolder: if subfolder:
file_filter = f'{subfolder}/*' file_filter = f'{subfolder}/*'
if not os.path.exists(pretrained_model_name_or_path): if not os.path.exists(pretrained_model_name_or_path):
revision = kwargs.pop('revision', None) revision = _ms_revision(kwargs.pop('revision', None))
if revision is None or revision == 'main':
revision = 'master'
if file_filter is not None: if file_filter is not None:
allow_file_pattern = file_filter allow_file_pattern = file_filter
local_files_only = kwargs.pop('local_files_only', False) download_kwargs = _ms_download_kwargs_from_hf(
kwargs, revision=revision)
model_dir = snapshot_download( model_dir = snapshot_download(
pretrained_model_name_or_path, pretrained_model_name_or_path,
revision=revision,
local_files_only=local_files_only,
ignore_file_pattern=ignore_file_pattern, ignore_file_pattern=ignore_file_pattern,
allow_file_pattern=allow_file_pattern) allow_file_pattern=allow_file_pattern,
**download_kwargs)
if subfolder: if subfolder:
model_dir = os.path.join(model_dir, subfolder) model_dir = os.path.join(model_dir, subfolder)
else: else:
@@ -592,11 +659,6 @@ def _unpatch_kernels():
del kernels_utils._get_hf_api_origin del kernels_utils._get_hf_api_origin
def _ms_revision(revision):
"""Translate an HF revision string into one ModelScope accepts."""
return 'master' if revision in (None, 'main') else revision
class _MsKernelApi: class _MsKernelApi:
"""Minimal `HfApi` look-alike that forwards to ModelScope. Only the """Minimal `HfApi` look-alike that forwards to ModelScope. Only the
handful of methods that `kernels` actually calls are implemented. handful of methods that `kernels` actually calls are implemented.

View File

@@ -374,6 +374,30 @@ def create_module_from_files(file_list, file_prefix, module_name):
importlib.invalidate_caches() importlib.invalidate_caches()
def _module_name_from_model_dir(model_dir: str) -> str:
"""Derive a valid Python package name from a local model directory.
modelscope_hub 0.1.x returns paths like
``{cache}/models/{owner}--{name}/snapshots/{revision}``. Using
``Path(model_dir).stem`` on a revision such as ``v1.0.4`` yields
``v1.0``, and ``importlib.import_module('v1.0.xxx')`` fails with
``ModuleNotFoundError: No module named 'v1'`` because dots are package
separators. Prefer the ``{owner}--{name}`` segment (plus revision for
isolation) and sanitize to a valid identifier.
"""
path = Path(model_dir).resolve()
if path.parent.name == 'snapshots':
base = f'{path.parent.parent.name}__{path.name}'
else:
# Use the full directory name, not .stem, so dotted names are kept
# intact before sanitization (stem would turn ``v1.0.4`` into ``v1.0``).
base = path.name
module_name = re.sub(r'[^0-9A-Za-z_]', '_', base)
if not module_name or module_name[0].isdigit():
module_name = f'm_{module_name}'
return module_name
def import_module_from_model_dir(model_dir): def import_module_from_model_dir(model_dir):
""" import all the necessary module from a model dir """ import all the necessary module from a model dir
@@ -383,7 +407,6 @@ def import_module_from_model_dir(model_dir):
No returns, raise error if failed No returns, raise error if failed
""" """
from pathlib import Path
file_scanner = FilesAstScanning() file_scanner = FilesAstScanning()
file_scanner.traversal_files(model_dir, include_init=True) file_scanner.traversal_files(model_dir, include_init=True)
file_dirs = file_scanner.file_dirs file_dirs = file_scanner.file_dirs
@@ -395,7 +418,7 @@ def import_module_from_model_dir(model_dir):
if BASE_MODULE_DIR not in sys.path: if BASE_MODULE_DIR not in sys.path:
sys.path.append(BASE_MODULE_DIR) sys.path.append(BASE_MODULE_DIR)
module_name = Path(model_dir).stem module_name = _module_name_from_model_dir(model_dir)
# in order to keep forward compatibility, we add module path to # in order to keep forward compatibility, we add module path to
# sys.path so that submodule can be imported directly as before # sys.path so that submodule can be imported directly as before

View File

@@ -1,5 +1,5 @@
filelock filelock
modelscope-hub>=0.0.7 modelscope-hub>=0.1.8
packaging packaging
requests>=2.25 requests>=2.25
setuptools setuptools

View File

@@ -1,5 +1,6 @@
import tempfile import tempfile
import unittest import unittest
from unittest import mock
from tqdm import tqdm from tqdm import tqdm
@@ -48,5 +49,36 @@ class ProgressCallbackTest(unittest.TestCase):
print(f'model_dir: {model_dir}') print(f'model_dir: {model_dir}')
class SnapshotDownloadForwardTest(unittest.TestCase):
"""Network-free tests: the shim forwards progress_callbacks to compat."""
# ``modelscope.hub.snapshot_download`` the attribute is shadowed by the
# re-exported function, so patch via the fully qualified module string.
_COMPAT_TARGET = \
'modelscope.hub.snapshot_download._compat_snapshot_download'
def test_progress_callbacks_forwarded_to_compat(self):
from modelscope.hub.snapshot_download import snapshot_download
with mock.patch(
self._COMPAT_TARGET, return_value='/tmp/snapshot') as m:
result = snapshot_download(
'owner/repo', progress_callbacks=[NewProgressCallback])
self.assertEqual(result, '/tmp/snapshot')
_, kwargs = m.call_args
self.assertEqual(kwargs['progress_callbacks'], [NewProgressCallback])
def test_progress_callbacks_default_none(self):
from modelscope.hub.snapshot_download import snapshot_download
with mock.patch(
self._COMPAT_TARGET, return_value='/tmp/snapshot') as m:
snapshot_download('owner/repo')
_, kwargs = m.call_args
self.assertIsNone(kwargs['progress_callbacks'])
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

View File

@@ -0,0 +1,54 @@
import importlib
import sys
import unittest
from unittest import mock
class LegacyCacheGuardTest(unittest.TestCase):
"""Tests for the modelscope-hub capability guard in the shim.
Network-free: only probes whether the loaded modelscope-hub exposes the
legacy-cache auto-detection capability and warns once when it does not.
"""
def setUp(self):
# NOTE: ``modelscope.hub.snapshot_download`` the *attribute* is shadowed
# by the re-exported function in ``modelscope.hub.__init__``; use
# importlib to obtain the actual submodule object.
sd = importlib.import_module('modelscope.hub.snapshot_download')
self.sd = sd
# Reset the fire-once probe cache before each test.
sd._legacy_cache_capability = None
def tearDown(self):
self.sd._legacy_cache_capability = None
def test_capability_present_no_warning(self):
sd = self.sd
# The real modelscope-hub DownloadManager has _find_legacy_repo_dir.
with mock.patch.object(sd.logger, 'warning') as warn:
sd._warn_if_legacy_cache_detection_unavailable()
self.assertTrue(sd._legacy_cache_capability)
warn.assert_not_called()
def test_capability_absent_warns_once(self):
sd = self.sd
class _OldDownloadManager: # lacks _find_legacy_repo_dir
pass
fake_module = mock.MagicMock()
fake_module.DownloadManager = _OldDownloadManager
with mock.patch.dict(sys.modules,
{'modelscope_hub._download': fake_module}):
with mock.patch.object(sd.logger, 'warning') as warn:
sd._warn_if_legacy_cache_detection_unavailable()
sd._warn_if_legacy_cache_detection_unavailable()
self.assertFalse(sd._legacy_cache_capability)
self.assertEqual(warn.call_count, 1) # fire-once
if __name__ == '__main__':
unittest.main()

View File

@@ -0,0 +1,103 @@
# Copyright (c) Alibaba, Inc. and its affiliates.
import os
import tempfile
import unittest
from pathlib import Path
from unittest import mock
from modelscope.hub.utils.utils import find_reusable_legacy_repo_dir
class LegacyCacheReuseTest(unittest.TestCase):
"""Old flat/hub cache layouts should be reusable without re-download."""
def setUp(self):
self._tmpdir = tempfile.TemporaryDirectory()
self.cache = Path(self._tmpdir.name)
self.model_id = 'iic/nlp_xlmr_named-entity-recognition_eng-ecommerce-query'
self.owner, self.name = self.model_id.split('/', 1)
def tearDown(self):
self._tmpdir.cleanup()
def _touch_model_dir(self, path: Path):
path.mkdir(parents=True, exist_ok=True)
(path / 'configuration.json').write_text('{}', encoding='utf-8')
def test_finds_flat_legacy_cache(self):
legacy = self.cache / self.owner / self.name
self._touch_model_dir(legacy)
found = find_reusable_legacy_repo_dir(
self.model_id, cache_dir=self.cache)
self.assertEqual(found, str(legacy))
def test_finds_hub_legacy_cache(self):
legacy = self.cache / 'hub' / self.owner / self.name
self._touch_model_dir(legacy)
found = find_reusable_legacy_repo_dir(
self.model_id, cache_dir=self.cache)
self.assertEqual(found, str(legacy))
def test_reuses_unsafed_models_slash_layout(self):
# Hub only probes safe_name (dots -> ___); unsafed path is reusable.
dotted_id = 'org/model.with.dots'
owner, name = dotted_id.split('/', 1)
slash = self.cache / 'models' / owner / name
self._touch_model_dir(slash)
found = find_reusable_legacy_repo_dir(dotted_id, cache_dir=self.cache)
self.assertEqual(found, str(slash))
def test_prefers_hub_known_safe_slash_layout(self):
dotted_id = 'org/model.with.dots'
owner, name = dotted_id.split('/', 1)
safe = self.cache / 'models' / owner / name.replace('.', '___')
flat = self.cache / owner / name
self._touch_model_dir(safe)
self._touch_model_dir(flat)
found = find_reusable_legacy_repo_dir(dotted_id, cache_dir=self.cache)
self.assertIsNone(found)
def test_prefers_hub_known_owner_dash_layout(self):
modern = self.cache / 'models' / self.model_id.replace('/', '--')
flat = self.cache / self.owner / self.name
self._touch_model_dir(modern)
self._touch_model_dir(flat)
found = find_reusable_legacy_repo_dir(
self.model_id, cache_dir=self.cache)
self.assertIsNone(found)
def test_empty_legacy_dir_ignored(self):
(self.cache / self.owner / self.name).mkdir(parents=True)
found = find_reusable_legacy_repo_dir(
self.model_id, cache_dir=self.cache)
self.assertIsNone(found)
def test_uses_modelscope_cache_env(self):
legacy = self.cache / self.owner / self.name
self._touch_model_dir(legacy)
with mock.patch.dict(os.environ,
{'MODELSCOPE_CACHE': str(self.cache)}):
found = find_reusable_legacy_repo_dir(self.model_id)
self.assertEqual(found, str(legacy))
def test_default_root_matches_hub_not_sdk_hub_suffix(self):
# Without MODELSCOPE_CACHE, hub uses ~/.cache/modelscope (no /hub).
modern = (
Path.home() / '.cache' / 'modelscope' / 'models'
/ self.model_id.replace('/', '--'))
# Do not create real home dirs; patch the hub root helper instead.
with mock.patch(
'modelscope.hub.utils.utils._modelscope_hub_cache_root',
return_value=self.cache):
modern_under_test = (
self.cache / 'models' / self.model_id.replace('/', '--'))
flat = self.cache / self.owner / self.name
self._touch_model_dir(modern_under_test)
self._touch_model_dir(flat)
found = find_reusable_legacy_repo_dir(self.model_id)
self.assertIsNone(found)
self.assertFalse(modern.exists()) # we never touched real home cache
if __name__ == '__main__':
unittest.main()

View File

@@ -249,19 +249,303 @@ class HFUtilTest(unittest.TestCase):
f'Expected no weight files in {model_dir}, but found: ' f'Expected no weight files in {model_dir}, but found: '
f"{[f for f in files if f.endswith('.safetensors') or f.endswith('.bin')]}" f"{[f for f in files if f.endswith('.safetensors') or f.endswith('.bin')]}"
) )
cache_dir = os.path.dirname(model_dir) # modelscope_hub 0.1.x layout uses models/{owner}--{name}/...;
cache_dir = os.path.dirname(cache_dir) # older layout used {owner}/{name}/. Accept either.
model_dir_2 = os.path.join(cache_dir, 'nomic-ai', 'nomic-bert-2048') cache_root = model_dir
if os.path.exists(model_dir_2): for _ in range(4):
files = os.listdir(model_dir_2) parent = os.path.dirname(cache_root)
has_weight_files = any( if parent == cache_root:
f.endswith('.safetensors') or f.endswith('.bin') break
for f in files) cache_root = parent
self.assertFalse( candidates = [
has_weight_files, os.path.join(cache_root, 'nomic-ai', 'nomic-bert-2048'),
f'Expected no weight files in {model_dir}, but found: ' os.path.join(cache_root, 'models',
f"{[f for f in files if f.endswith('.safetensors') or f.endswith('.bin')]}" 'nomic-ai--nomic-bert-2048'),
) ]
for model_dir_2 in candidates:
if not os.path.exists(model_dir_2):
continue
# Walk into snapshots/{rev} if present.
check_dirs = [model_dir_2]
snapshots = os.path.join(model_dir_2, 'snapshots')
if os.path.isdir(snapshots):
check_dirs.extend(
os.path.join(snapshots, d)
for d in os.listdir(snapshots)
if os.path.isdir(os.path.join(snapshots, d)))
for check_dir in check_dirs:
files = os.listdir(check_dir)
has_weight_files = any(
f.endswith('.safetensors') or f.endswith('.bin')
for f in files)
self.assertFalse(
has_weight_files,
f'Expected no weight files in {check_dir}, but found: '
f"{[f for f in files if f.endswith('.safetensors') or f.endswith('.bin')]}"
)
def test_dynamic_module_double_dash_cache_path(self):
"""Cross-repo auto_map must survive cache paths that contain '--'.
modelscope_hub 0.1.x stores repos under ``models/{owner}--{name}/``.
Rejoining that path into ``class_reference`` with ``--`` makes
transformers' ``split("--")`` raise ValueError.
"""
from unittest import mock
from modelscope.utils.hf_util.patcher import \
_get_class_from_dynamic_module
tmp = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, tmp, ignore_errors=True)
local_path = os.path.join(tmp, 'models', 'nomic-ai--nomic-bert-2048',
'snapshots', 'rev')
os.makedirs(local_path)
pretrained = os.path.join(tmp, 'models',
'nomic-ai--nomic-embed-text-v1.5',
'snapshots', 'rev')
os.makedirs(pretrained)
captured = {}
def fake_origin(class_reference, pretrained_model_name_or_path, *args,
**kwargs):
# Signature must match transformers so has_pretrained_arg is True.
captured['class_reference'] = class_reference
captured['pretrained'] = pretrained_model_name_or_path
return type('DummyConfig', (), {})
class_ref = ('nomic-ai/nomic-bert-2048--'
'configuration_hf_nomic_bert.NomicBertConfig')
# create=True: do not permanently leave origin_* on the module
# (would break test_import_not_pollute_dynamic_module).
with mock.patch(
'transformers.dynamic_module_utils.origin_get_class_from_dynamic_module',
new=fake_origin,
create=True):
with mock.patch(
'modelscope.snapshot_download', return_value=local_path):
_get_class_from_dynamic_module(class_ref, pretrained)
# Must pass bare module.Class (no '--') so transformers does not split.
self.assertEqual(captured['class_reference'],
'configuration_hf_nomic_bert.NomicBertConfig')
# Local cache path (which contains '--') is pretrained_model_name_or_path.
self.assertEqual(captured['pretrained'], local_path)
def test_dynamic_module_remote_pretrained_tuple_args(self):
"""Remote pretrained_model_name_or_path must not mutate args in place.
``*args`` is a tuple; ``args[0] = snapshot_download(...)`` raises
TypeError. Rebuild the tuple instead (regression from 9379504f).
"""
from unittest import mock
from modelscope.utils.hf_util.patcher import \
_get_class_from_dynamic_module
tmp = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, tmp, ignore_errors=True)
downloaded = os.path.join(tmp, 'models', 'org--model', 'snapshots',
'rev')
os.makedirs(downloaded)
captured = {}
def fake_origin(class_reference, pretrained_model_name_or_path, *args,
**kwargs):
captured['class_reference'] = class_reference
captured['pretrained'] = pretrained_model_name_or_path
return type('DummyConfig', (), {})
# No '--' in class_reference: only the pretrained download branch runs.
remote_id = 'org/model-not-on-disk'
with mock.patch(
'transformers.dynamic_module_utils.origin_get_class_from_dynamic_module',
new=fake_origin,
create=True):
with mock.patch(
'modelscope.snapshot_download',
return_value=downloaded) as sd:
# Must not raise TypeError: 'tuple' object does not support
# item assignment.
_get_class_from_dynamic_module('modeling.Foo', remote_id)
sd.assert_called_once_with(remote_id, local_files_only=False)
self.assertEqual(captured['class_reference'], 'modeling.Foo')
self.assertEqual(captured['pretrained'], downloaded)
def test_dynamic_module_local_files_only_forwarded(self):
"""Download kwargs must be forwarded to both snapshot_download calls.
Cross-repo auto_map references previously omitted local_files_only /
cache_dir / token / code_revision, so offline and custom-cache loads
still hit the wrong download path for the referenced repo.
"""
from unittest import mock
from modelscope.utils.hf_util.patcher import \
_get_class_from_dynamic_module
tmp = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, tmp, ignore_errors=True)
downloaded = os.path.join(tmp, 'models', 'org--model', 'snapshots',
'rev')
cross_repo = os.path.join(tmp, 'models', 'org--other', 'snapshots',
'rev')
os.makedirs(downloaded)
os.makedirs(cross_repo)
def fake_origin(class_reference, pretrained_model_name_or_path, *args,
**kwargs):
return type('DummyConfig', (), {})
remote_id = 'org/model-not-on-disk'
class_ref = 'org/other--configuration_foo.FooConfig'
call_kwargs = []
cache_dir = os.path.join(tmp, 'custom_cache')
token = 'ms-test-token'
def fake_download(repo_id, **kwargs):
call_kwargs.append((repo_id, dict(kwargs)))
if repo_id == remote_id:
return downloaded
if repo_id == 'org/other':
return cross_repo
raise AssertionError(f'unexpected download: {repo_id}')
with mock.patch(
'transformers.dynamic_module_utils.origin_get_class_from_dynamic_module',
new=fake_origin,
create=True):
with mock.patch(
'modelscope.snapshot_download', side_effect=fake_download):
_get_class_from_dynamic_module(
class_ref,
pretrained_model_name_or_path=remote_id,
local_files_only=True,
cache_dir=cache_dir,
token=token,
revision='model-rev',
code_revision='code-rev')
self.assertEqual(len(call_kwargs), 2)
self.assertEqual(call_kwargs[0][0], remote_id)
self.assertEqual(
call_kwargs[0][1], {
'local_files_only': True,
'cache_dir': cache_dir,
'token': token,
'revision': 'model-rev',
})
self.assertEqual(call_kwargs[1][0], 'org/other')
self.assertEqual(call_kwargs[1][1]['local_files_only'], True)
self.assertEqual(call_kwargs[1][1]['cache_dir'], cache_dir)
self.assertEqual(call_kwargs[1][1]['token'], token)
self.assertEqual(call_kwargs[1][1]['revision'], 'code-rev')
self.assertIn('ignore_file_pattern', call_kwargs[1][1])
def test_ms_download_kwargs_from_hf(self):
"""Shared HF→MS download kwargs mapping used by patcher download paths."""
from modelscope.utils.hf_util.patcher import _ms_download_kwargs_from_hf
self.assertEqual(
_ms_download_kwargs_from_hf({}), {'local_files_only': False})
got = _ms_download_kwargs_from_hf(
{
'local_files_only': True,
'cache_dir': '/tmp/c',
'token': 'sekrit',
'token_ignored': True,
},
revision='main')
self.assertEqual(
got, {
'local_files_only': True,
'cache_dir': '/tmp/c',
'token': 'sekrit',
'revision': 'master',
})
# HF token=True means "default creds"; only string tokens are forwarded.
self.assertNotIn('token', _ms_download_kwargs_from_hf({'token': True}))
def test_get_model_dir_forwards_cache_dir_and_token(self):
"""from_pretrained download path must forward cache_dir and token."""
from unittest import mock
from modelscope import AutoConfig
tmp = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, tmp, ignore_errors=True)
with open(os.path.join(tmp, 'config.json'), 'w') as f:
f.write('{"model_type": "bert", "hidden_size": 8}')
cache_dir = os.path.join(tmp, 'custom_cache')
token = 'ms-test-token'
with mock.patch(
'modelscope.snapshot_download', return_value=tmp) as sd:
AutoConfig.from_pretrained(
'org/model-not-on-disk',
cache_dir=cache_dir,
token=token,
local_files_only=True)
sd.assert_called_once()
_, kwargs = sd.call_args
self.assertEqual(kwargs.get('cache_dir'), cache_dir)
self.assertEqual(kwargs.get('token'), token)
self.assertTrue(kwargs.get('local_files_only'))
def test_dynamic_module_pretrained_via_kwargs(self):
"""pretrained_model_name_or_path may be passed as a keyword argument."""
from unittest import mock
from modelscope.utils.hf_util.patcher import \
_get_class_from_dynamic_module
tmp = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, tmp, ignore_errors=True)
downloaded = os.path.join(tmp, 'models', 'org--model', 'snapshots',
'rev')
cross_repo = os.path.join(tmp, 'models', 'org--other', 'snapshots',
'rev')
os.makedirs(downloaded)
os.makedirs(cross_repo)
captured = {}
def fake_origin(class_reference, pretrained_model_name_or_path, *args,
**kwargs):
captured['class_reference'] = class_reference
captured['pretrained'] = pretrained_model_name_or_path
captured['kwargs'] = kwargs
return type('DummyConfig', (), {})
remote_id = 'org/model-not-on-disk'
class_ref = 'org/other--configuration_foo.FooConfig'
def fake_download(repo_id, **kwargs):
if repo_id == remote_id:
return downloaded
if repo_id == 'org/other':
return cross_repo
raise AssertionError(f'unexpected download: {repo_id}')
with mock.patch(
'transformers.dynamic_module_utils.origin_get_class_from_dynamic_module',
new=fake_origin,
create=True):
with mock.patch(
'modelscope.snapshot_download', side_effect=fake_download):
# Keyword form: must download and not pass duplicate positional.
_get_class_from_dynamic_module(
class_ref, pretrained_model_name_or_path=remote_id)
self.assertEqual(captured['class_reference'],
'configuration_foo.FooConfig')
self.assertEqual(captured['pretrained'], cross_repo)
self.assertNotIn('pretrained_model_name_or_path', captured['kwargs'])
def test_import_not_pollute_dynamic_module(self): def test_import_not_pollute_dynamic_module(self):
"""Importing from modelscope must not globally patch """Importing from modelscope must not globally patch

View File

@@ -0,0 +1,30 @@
# Copyright (c) Alibaba, Inc. and its affiliates.
import unittest
from modelscope.utils.automodel_utils import check_model_from_owner_group
class OwnerGroupPathSafetyTest(unittest.TestCase):
"""Safety checks for trusted-owner cache path recognition."""
def test_empty_name_cache_path_rejected(self):
# modelscope_hub layout: {cache}/{owner}--{name}/snapshots/{rev}
# Empty name ("iic--") must not be treated as a trusted owner path.
self.assertFalse(
check_model_from_owner_group('/cache/iic--/snapshots/v1'))
self.assertFalse(
check_model_from_owner_group('/cache/damo--/snapshots/v1'))
def test_valid_and_spoof_cache_paths(self):
self.assertTrue(
check_model_from_owner_group('/cache/iic--x/snapshots/v1'))
self.assertFalse(
check_model_from_owner_group('/cache/--iic/snapshots/v1'))
self.assertFalse(
check_model_from_owner_group(
'/cache/iic--hacked--evil/snapshots/v1'))
self.assertTrue(check_model_from_owner_group('/cache/iic/some_model'))
if __name__ == '__main__':
unittest.main()

View File

@@ -127,5 +127,40 @@ class PluginTest(unittest.TestCase):
self.assertEqual(len(result.items()), len(OFFICIAL_PLUGINS)) self.assertEqual(len(result.items()), len(OFFICIAL_PLUGINS))
class ModuleNameFromModelDirTest(unittest.TestCase):
"""Regression: snapshot revision paths must not become dotted module names."""
def test_snapshot_revision_with_dots(self):
from modelscope.utils.plugins import _module_name_from_model_dir
tmp = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, tmp, ignore_errors=True)
model_dir = os.path.join(tmp, 'models',
'iic--cv_unet_skin_retouching_torch',
'snapshots', 'v1.0.4')
os.makedirs(model_dir)
name = _module_name_from_model_dir(model_dir)
self.assertNotIn('.', name)
self.assertTrue(
name.startswith('iic__cv_unet_skin_retouching_torch__v1_0_4'))
# Old Path(model_dir).stem produced 'v1.0', which importlib treats as
# package 'v1' → ModuleNotFoundError: No module named 'v1'.
self.assertNotEqual(name, 'v1.0')
self.assertFalse(name.startswith('v1'))
def test_flat_cache_layout_unchanged(self):
from modelscope.utils.plugins import _module_name_from_model_dir
tmp = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, tmp, ignore_errors=True)
model_dir = os.path.join(tmp, 'iic', 'cv_unet_skin_retouching_torch')
os.makedirs(model_dir)
self.assertEqual(
_module_name_from_model_dir(model_dir),
'cv_unet_skin_retouching_torch')
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()