From 0fd381159aa60f43e9c32007e690773a19b61b54 Mon Sep 17 00:00:00 2001 From: suluyana <110878454+suluyana@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:16:54 +0800 Subject: [PATCH] Backport/release 1.39 to master (#1767) --- .dev_scripts/ci_container_test.sh | 3 +- .github/workflows/docker-image.yml | 2 +- docker/Dockerfile.amd | 26 ++ docker/Dockerfile.ascend | 26 +- docker/Dockerfile.ubuntu | 19 +- docker/build_image.py | 396 +++++++++++++++++++- docker/scripts/modelscope_env_init.sh | 1 - modelscope/hub/file_download.py | 39 +- modelscope/hub/snapshot_download.py | 64 +++- modelscope/hub/utils/utils.py | 72 ++++ modelscope/utils/automodel_utils.py | 9 +- modelscope/utils/hf_util/patcher.py | 104 +++-- modelscope/utils/plugins.py | 27 +- requirements/hub.txt | 2 +- tests/hub/test_download_callback.py | 32 ++ tests/hub/test_legacy_cache_guard.py | 54 +++ tests/hub/test_legacy_cache_reuse.py | 103 +++++ tests/utils/test_hf_util.py | 310 ++++++++++++++- tests/utils/test_owner_group_path_safety.py | 30 ++ tests/utils/test_plugin.py | 35 ++ 20 files changed, 1283 insertions(+), 71 deletions(-) create mode 100644 docker/Dockerfile.amd create mode 100644 tests/hub/test_legacy_cache_guard.py create mode 100644 tests/hub/test_legacy_cache_reuse.py create mode 100644 tests/utils/test_owner_group_path_safety.py diff --git a/.dev_scripts/ci_container_test.sh b/.dev_scripts/ci_container_test.sh index 8c541fc5..886749ab 100644 --- a/.dev_scripts/ci_container_test.sh +++ b/.dev_scripts/ci_container_test.sh @@ -1,8 +1,7 @@ if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/ - pip config set global.extra-index-url https://pypi.org/simple/ pip config set install.trusted-host mirrors.aliyun.com - pip install -r requirements/tests.txt + PIP_EXTRA_INDEX_URL=https://pypi.org/simple pip install -r requirements/tests.txt git config --global --add safe.directory /Maas-lib git config --global user.email tmp git config --global user.name tmp.com diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index f5c576da..364d1834 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -11,7 +11,7 @@ on: description: 'ModelScope branch to build from(release/x.xx)' required: true image_type: - description: 'The image type to build(base/old/stable/latest)' + description: 'The image type to build(base/old/stable/latest/amd/ascend)' required: true modelscope_version: description: 'ModelScope version to use(x.xx.x)' diff --git a/docker/Dockerfile.amd b/docker/Dockerfile.amd new file mode 100644 index 00000000..ae6796f0 --- /dev/null +++ b/docker/Dockerfile.amd @@ -0,0 +1,26 @@ +FROM {base_image} + +ARG BASE_IMAGE_TAG={base_image_tag} +LABEL modelscope.base_image="vllm/vllm-openai-rocm:${BASE_IMAGE_TAG}" + +# Build-time only (ARG does not persist into the image). Aliyun mirror can lag PyPI (~1h). +ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple + +COPY docker/scripts/modelscope_env_init.sh /usr/local/bin/ms_env_init.sh + +ARG CUR_TIME={cur_time} +RUN echo "CUR_TIME=${CUR_TIME}" && echo "BASE_IMAGE_TAG=${BASE_IMAGE_TAG}" + +ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple +RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \ + pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \ + pip config set install.trusted-host mirrors.aliyun.com && \ + cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b {modelscope_branch} --single-branch https://github.com/modelscope/modelscope.git && \ + cd modelscope && pip install --no-cache-dir . -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ + cd / && rm -fr /tmp/modelscope && pip cache purge + +ENV VLLM_USE_MODELSCOPE=True +ENV LMDEPLOY_USE_MODELSCOPE=True +ENV MODELSCOPE_CACHE=/mnt/workspace/.cache/modelscope/hub + +SHELL ["/bin/bash", "-c"] diff --git a/docker/Dockerfile.ascend b/docker/Dockerfile.ascend index d15548b6..6bf2e958 100644 --- a/docker/Dockerfile.ascend +++ b/docker/Dockerfile.ascend @@ -5,6 +5,8 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ PIP_RETRIES=10 \ SOC_VERSION={soc_version} \ CANN_VERSION={cann_version} +# Build-time only (ARG does not persist into the image). +ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple SHELL ["/bin/bash", "-c"] @@ -36,12 +38,7 @@ RUN set -eux; \ esac RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \ - pip config set global.extra-index-url "https://pypi.org/simple" && \ - pip config set install.trusted-host mirrors.aliyun.com && \ - ARCH=$(uname -m) && \ - if [ "$ARCH" = "x86_64" ]; then \ - pip config set global.extra-index-url "https://pypi.org/simple https://download.pytorch.org/whl/cpu/"; \ - fi + pip config set install.trusted-host mirrors.aliyun.com {extra_content} # ---------- Install vllm + vllm-ascend ---------- @@ -51,6 +48,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ git clone --depth 1 --branch {vllm_ascend_git_ref} https://github.com/vllm-project/vllm-ascend.git RUN ARCH=$(uname -m) && \ + export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \ source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ source /usr/local/Ascend/nnal/atb/set_env.sh && \ # Install torch & torch_npu & torchvision @@ -66,7 +64,8 @@ RUN git clone --depth 1 --branch {megatron_branch} https://github.com/NVIDIA/Meg GIT_LFS_SKIP_SMUDGE=1 git clone --depth 1 -b {swift_branch} --single-branch https://github.com/modelscope/ms-swift.git /ms-swift # ---------- Install training-side repositories ---------- -RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ +RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \ + source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \ cd /MindSpeed && pip install --no-cache-dir -e . && \ pip install --no-cache-dir mcore-bridge -i https://pypi.org/simple/ -U && \ @@ -75,7 +74,8 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ # ---------- Pin torch to the correct version + torch_npu ---------- # x86: must force-install the CPU build from pytorch.org/whl/cpu # aarch64: PyPI only provides the CPU build, so install it directly from the Aliyun mirror -RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ +RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \ + source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \ ARCH=$(uname -m) && \ if [ "$ARCH" = "x86_64" ]; then \ @@ -99,7 +99,8 @@ ENV PYTHONPATH=/Megatron-LM:${PYTHONPATH} # install dependencies COPY requirements /var/modelscope -RUN pip uninstall ms-swift modelscope -y && pip install --no-cache-dir pip==23.* -U && \ +RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \ + pip uninstall ms-swift modelscope -y && pip install --no-cache-dir pip==23.* -U && \ if [ "$INSTALL_MS_DEPS" = "True" ]; then \ pip install --no-cache-dir omegaconf==2.0.6 && \ pip install 'editdistance==0.8.1' && \ @@ -125,9 +126,11 @@ fi ARG CUR_TIME={cur_time} RUN echo $CUR_TIME -RUN pip install --no-cache-dir --no-build-isolation OpenCC +RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \ + pip install --no-cache-dir --no-build-isolation OpenCC -RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ +RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \ + source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \ pip install --no-cache-dir -U funasr scikit-learn && \ pip install --no-cache-dir -U qwen_vl_utils qwen_omni_utils librosa 'timm>=0.9.0' transformers accelerate peft trl safetensors && \ @@ -150,6 +153,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ # ---------- Install triton-ascend ---------- RUN set -eux; \ + export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}"; \ pip uninstall -y triton || true; \ pip uninstall -y triton-ascend || true; \ pip install --no-cache-dir --force-reinstall \ diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu index 2493f0e7..f9f1dbb4 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.ubuntu @@ -3,6 +3,8 @@ FROM {base_image} ARG DEBIAN_FRONTEND=noninteractive ENV TZ=Asia/Shanghai ENV arch=x86_64 +# Build-time only (ARG does not persist into the image). Aliyun mirror can lag PyPI (~1h). +ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple COPY docker/scripts/modelscope_env_init.sh /usr/local/bin/ms_env_init.sh RUN apt-get update && \ @@ -21,7 +23,9 @@ ARG IMAGE_TYPE={image_type} # install dependencies COPY requirements /var/modelscope -RUN pip uninstall ms-swift modelscope -y && pip --no-cache-dir install pip==23.* -U && \ +ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple +RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \ + pip uninstall ms-swift modelscope -y && pip --no-cache-dir install pip==23.* -U && \ if [ "$INSTALL_MS_DEPS" = "True" ]; then \ pip --no-cache-dir install omegaconf==2.0.6 && \ pip install 'editdistance==0.8.1' && \ @@ -34,7 +38,7 @@ if [ "$INSTALL_MS_DEPS" = "True" ]; then \ pip install --no-cache-dir 'scipy' && \ pip install --no-cache-dir funtextprocessing typeguard==2.13.3 scikit-learn -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ pip install --no-cache-dir 'decord>=0.6.0' mpi4py paint_ldm ipykernel fasttext -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ - pip install --no-cache-dir ipywidgets && \ + pip install --no-cache-dir ipywidgets jupyter_core nbconvert nbclient && \ pip install --no-cache-dir 'blobfile>=1.0.5' && \ pip uninstall MinDAEC -y && \ pip install https://modelscope.oss-cn-beijing.aliyuncs.com/releases/dependencies/MinDAEC-0.0.2-py3-none-any.whl && \ @@ -47,7 +51,9 @@ fi ARG CUR_TIME={cur_time} RUN echo $CUR_TIME -RUN bash /tmp/install.sh {version_args} && \ +ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple +RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \ + bash /tmp/install.sh {version_args} && \ pip install --no-cache-dir -U funasr scikit-learn && \ pip install --no-cache-dir -U qwen_vl_utils qwen_omni_utils librosa timm transformers accelerate peft trl safetensors && \ cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b {swift_branch} --single-branch https://github.com/modelscope/ms-swift.git && \ @@ -63,16 +69,17 @@ RUN bash /tmp/install.sh {version_args} && \ pip install --no-cache-dir transformers diffusers 'timm>=0.9.0' && pip cache purge; \ pip install --no-cache-dir omegaconf==2.3.0 && pip cache purge; \ pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \ - pip config set global.extra-index-url https://pypi.org/simple && \ pip config set install.trusted-host mirrors.aliyun.com && \ cp /tmp/resources/ubuntu2204.aliyun /etc/apt/sources.list -RUN if [ "$IMAGE_TYPE" = "gpu" ]; then \ +ARG PIP_EXTRA_INDEX_URL=https://pypi.org/simple +RUN export PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" && \ + if [ "$IMAGE_TYPE" = "gpu" ]; then \ pip install --no-cache-dir math_verify "gradio<5.33" "deepspeed<0.19" ray -U && \ pip install --no-cache-dir mcore-bridge -i https://pypi.org/simple/ -U && \ pip install --no-cache-dir liger_kernel wandb swanlab nvitop pre-commit "transformers<5.9" "trl<1.0" "peft<0.20" huggingface-hub -U && \ - pip install --no-cache-dir --no-build-isolation transformer_engine[pytorch]; \ + pip install --no-cache-dir --no-build-isolation "transformer_engine[pytorch]==2.16.0"; \ cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone https://github.com/NVIDIA/apex && \ cd apex && pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" ./ && \ cd / && rm -fr /tmp/apex && pip cache purge; \ diff --git a/docker/build_image.py b/docker/build_image.py index 18c1f2dd..f5df8262 100644 --- a/docker/build_image.py +++ b/docker/build_image.py @@ -3,14 +3,27 @@ import os import platform import re import subprocess +import urllib.error +import urllib.request from copy import copy from datetime import datetime -from typing import Any +from typing import Any, List, Optional + +import json docker_registry = os.environ['DOCKER_REGISTRY'] assert docker_registry, 'You must pass a valid DOCKER_REGISTRY' timestamp = datetime.now() formatted_time = timestamp.strftime('%Y%m%d%H%M%S') +VLLM_ROCM_REPO = 'vllm/vllm-openai-rocm' +_FLOATING_ROCM_TAGS = frozenset({ + 'latest', + 'latest-base', + 'nightly', + 'base-nightly', +}) +_VERSION_TAG_PATTERN = re.compile(r'^v\d+(?:\.\d+)*$') +_NIGHTLY_HASH_PATTERN = re.compile(r'^(?:base-)?nightly-[0-9a-f]{7,40}$') class Builder: @@ -359,7 +372,8 @@ class StableGPUImageBuilder(Builder): extra_content = extra_content.replace('{python_version}', self.args.python_version) extra_content += """ -RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy +RUN export PIP_EXTRA_INDEX_URL=https://pypi.org/simple && \ + pip install --no-cache-dir -U icecream soundfile pybind11 py-spy """ version_args = ( f'{self.args.torch_version} {self.args.torchvision_version} {self.args.torchaudio_version} ' @@ -420,7 +434,8 @@ class LatestGPUImageBuilder(StableGPUImageBuilder): extra_content = extra_content.replace('{python_version}', self.args.python_version) extra_content += """ -RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy +RUN export PIP_EXTRA_INDEX_URL=https://pypi.org/simple && \ + pip install --no-cache-dir -U icecream soundfile pybind11 py-spy """ version_args = ( f'{self.args.torch_version} {self.args.torchvision_version} {self.args.torchaudio_version} ' @@ -467,6 +482,370 @@ RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy return self.run_cmd('docker', 'push', image_tag2) +class AmdImageBuilder(Builder): + """Build ModelScope image on top of vllm/vllm-openai-rocm.""" + + @staticmethod + def _is_specific_release_tag(tag: str) -> bool: + tag = tag.strip() + if not tag or tag.lower() in _FLOATING_ROCM_TAGS: + return False + if tag.endswith('-base'): + return False + if _NIGHTLY_HASH_PATTERN.fullmatch(tag): + return False + return bool(_VERSION_TAG_PATTERN.fullmatch(tag)) + + @staticmethod + def _image_digest(tag_info: dict) -> Optional[str]: + digest = tag_info.get('digest') + if digest: + return digest + for image in tag_info.get('images') or []: + digest = image.get('digest') + if digest: + return digest + return None + + @classmethod + def _fetch_rocm_tags(cls, page_size: int = 100) -> List[dict]: + tags: List[dict] = [] + url = (f'https://hub.docker.com/v2/repositories/{VLLM_ROCM_REPO}/tags' + f'?page_size={page_size}&ordering=-last_updated') + while url: + req = urllib.request.Request( + url, headers={'User-Agent': 'modelscope-docker-builder'}) + try: + with urllib.request.urlopen(req, timeout=60) as resp: + payload = json.load(resp) + except (urllib.error.URLError, json.JSONDecodeError) as exc: + raise RuntimeError( + f'Failed to query Docker Hub tags for {VLLM_ROCM_REPO}: ' + f'{exc}') from exc + tags.extend(payload.get('results') or []) + url = payload.get('next') + # Only scan the first few pages; release tags are near the top. + if len(tags) >= 300: + break + if not tags: + raise RuntimeError( + f'No tags returned from Docker Hub for {VLLM_ROCM_REPO}') + return tags + + @classmethod + def resolve_latest_rocm_tag(cls) -> str: + """Resolve the newest concrete release tag for vllm-openai-rocm. + + Preference order: + 1. Semver tag (vX.Y.Z) that shares digest with floating ``latest`` + 2. Newest semver tag by Docker Hub ``last_updated`` + """ + tags = cls._fetch_rocm_tags() + by_name = {item['name']: item for item in tags if item.get('name')} + release_tags = [ + item for item in tags + if cls._is_specific_release_tag(item.get('name', '')) + ] + latest_info = by_name.get('latest') + latest_digest = cls._image_digest(latest_info) if latest_info else None + if latest_digest: + matched = [ + item for item in release_tags + if cls._image_digest(item) == latest_digest + ] + if matched: + # Prefer the first match in last_updated order from API. + chosen = matched[0]['name'] + print( + f'Resolved {VLLM_ROCM_REPO} latest digest to release tag: ' + f'{chosen}') + return chosen + + if not release_tags: + raise RuntimeError( + f'No concrete release tags found for {VLLM_ROCM_REPO}') + chosen = release_tags[0]['name'] + print(f'Resolved newest {VLLM_ROCM_REPO} release tag: {chosen}') + return chosen + + def init_args(self, args: Any) -> Any: + # Auto-discover from Docker Hub unless an explicit override is given. + override = getattr(args, 'base_image_tag', None) + if override and str(override).strip() and str( + override).strip().lower() not in {'auto', 'latest'}: + args.base_image_tag = str(override).strip() + if not self._is_specific_release_tag(args.base_image_tag): + raise ValueError( + 'base_image_tag override must be a concrete release tag ' + f'(e.g. v0.25.1), got: {args.base_image_tag}') + print(f'Using override AMD ROCm base image tag: ' + f'{args.base_image_tag}') + else: + args.base_image_tag = self.resolve_latest_rocm_tag() + if not args.base_image: + args.base_image = f'{VLLM_ROCM_REPO}:{args.base_image_tag}' + if not args.cuda_version: + args.cuda_version = '0.0.0' + return args + + @staticmethod + def _sanitize_tag(tag: str) -> str: + return re.sub(r'[^A-Za-z0-9._-]+', '-', tag) + + @staticmethod + def _normalize_version(version: str) -> str: + version = version.strip().lstrip('vV') + version = version.split('+')[0].split(' ')[0] + return re.sub(r'[^0-9A-Za-z._-]+', '', version) + + @staticmethod + def _python_tag_from_version(version: str) -> str: + parts = version.strip().split('.') + if len(parts) >= 2 and parts[0].isdigit() and parts[1].isdigit(): + return f'py{parts[0]}{parts[1]}' + return f'py{re.sub(r"[^0-9]", "", version)}' + + @classmethod + def _run_capture(cls, *cmd: str) -> subprocess.CompletedProcess: + return subprocess.run( + list(cmd), capture_output=True, text=True, check=False) + + @classmethod + def _probe_via_entrypoint(cls, base_image: str) -> dict: + """Read versions with docker run --entrypoint (no GPU required).""" + # Keep this script compact: it runs inside the base image via python -c. + script = ( + 'import json,os,pathlib,subprocess,sys\n' + 'info={"python":"%d.%d.%d"%sys.version_info[:3]}\n' + 'try:\n' + ' import torch\n' + ' info["torch"]=torch.__version__\n' + ' hip=getattr(torch.version,"hip",None)\n' + ' if hip: info["torch_hip"]=hip\n' + 'except Exception as e:\n' + ' info["torch_error"]=str(e)\n' + 'for p in ("/opt/rocm/.info/version","/opt/rocm/.info/version-dev"):\n' + ' f=pathlib.Path(p)\n' + ' if f.is_file():\n' + ' info["rocm_file"]=f.read_text().strip().splitlines()[0]\n' + ' break\n' + 'for k in ("ROCM_VERSION","HIP_VERSION","TORCH_VERSION"):\n' + ' if os.environ.get(k): info[k.lower()]=os.environ[k]\n' + 'def _dpkg_ver(*names):\n' + ' for n in names:\n' + ' try:\n' + ' r=subprocess.run(["dpkg-query","-W","-f=${Version}",n],' + 'capture_output=True,text=True)\n' + ' if r.returncode==0 and r.stdout.strip():\n' + ' return r.stdout.strip()\n' + ' except Exception:\n' + ' pass\n' + ' return None\n' + 'def _dpkg_scan(prefixes):\n' + ' try:\n' + ' r=subprocess.run(["dpkg-query","-W","-f=${Package}\\t${Version}\\n"],' + 'capture_output=True,text=True)\n' + ' except Exception:\n' + ' return {}\n' + ' found={}\n' + ' for line in (r.stdout or "").splitlines():\n' + ' if "\\t" not in line: continue\n' + ' pkg,ver=line.split("\\t",1)\n' + ' for pref in prefixes:\n' + ' if pkg==pref or pkg.startswith(pref+"-"):\n' + ' found.setdefault(pref,ver)\n' + ' return found\n' + 'pkgs=_dpkg_scan(("rccl","miopen"))\n' + 'info["system.library.rccl"]=_dpkg_ver("rccl") or pkgs.get("rccl")\n' + 'info["system.library.miopen"]=(' + '_dpkg_ver("miopen-hip","miopen") or pkgs.get("miopen"))\n' + 'print(json.dumps(info))\n') + for py in ('python3', 'python'): + result = cls._run_capture('docker', 'run', '--rm', '--network', + 'none', '--entrypoint', py, base_image, + '-c', script) + if result.returncode == 0 and result.stdout.strip(): + try: + return json.loads(result.stdout.strip().splitlines()[-1]) + except json.JSONDecodeError: + continue + return {} + + @classmethod + def _probe_via_history(cls, base_image: str) -> dict: + """Parse build ARGs from docker history (no container start).""" + result = cls._run_capture('docker', 'history', '--no-trunc', + '--format', '{{.CreatedBy}}', base_image) + if result.returncode != 0: + return {} + text = result.stdout + info = {} + for key, pattern in ( + ('rocm', r'ROCM_VERSION=([0-9]+(?:\.[0-9]+)*)'), + ('python', r'PYTHON_VERSION=([0-9]+(?:\.[0-9]+)*)'), + ('ubuntu', + r'org\.opencontainers\.image\.version=([0-9]+(?:\.[0-9]+)*)'), + ): + matches = re.findall(pattern, text) + if matches: + # docker history lists newest layers first. + info[key] = matches[0] + return info + + @classmethod + def _probe_via_create_cp(cls, base_image: str) -> dict: + """Copy version files out of a created (not started) container.""" + import tempfile + create = cls._run_capture('docker', 'create', base_image) + if create.returncode != 0: + return {} + cid = create.stdout.strip() + info = {} + try: + with tempfile.TemporaryDirectory() as tmp: + dest = os.path.join(tmp, 'version') + for src in ('/opt/rocm/.info/version', + '/opt/rocm/.info/version-dev'): + result = cls._run_capture('docker', 'cp', f'{cid}:{src}', + dest) + if result.returncode == 0 and os.path.isfile(dest): + with open(dest, 'r', encoding='utf-8') as f: + line = f.read().strip().splitlines() + if line: + info['rocm_file'] = line[0].strip() + break + finally: + cls._run_capture('docker', 'rm', '-f', cid) + return info + + @classmethod + def probe_base_image_versions(cls, base_image: str) -> dict: + """Discover rocm/python/torch without needing AMD GPU. + + Methods (in order): + 1. docker run --entrypoint python -c ... (CPU-only, no --device) + 2. docker history --no-trunc parse ROCM_VERSION/PYTHON_VERSION + 3. docker create + docker cp /opt/rocm/.info/version + """ + probed = {} + entry = cls._probe_via_entrypoint(base_image) + history = cls._probe_via_history(base_image) + copied = cls._probe_via_create_cp(base_image) + probed.update(history) + probed.update(copied) + probed.update(entry) + + rocm = ( + probed.get('rocm_file') or probed.get('rocm_version') + or probed.get('rocm') or probed.get('torch_hip') + or probed.get('hip_version')) + python_ver = probed.get('python') + torch_ver = probed.get('torch') or probed.get('torch_version') + ubuntu_ver = probed.get('ubuntu') + # Keep dpkg package versions as-is (may contain '~', e.g. 2.27.7.70201-81~22.04). + rccl_ver = probed.get('system.library.rccl') + miopen_ver = probed.get('system.library.miopen') + + versions = { + 'rocm': cls._normalize_version(rocm) if rocm else None, + 'python': + cls._normalize_version(python_ver) if python_ver else None, + 'torch': cls._normalize_version(torch_ver) if torch_ver else None, + 'ubuntu': + cls._normalize_version(ubuntu_ver) if ubuntu_ver else None, + 'system.library.rccl': rccl_ver or None, + 'system.library.miopen': miopen_ver or None, + } + print('Probed AMD base image versions:') + for key, value in versions.items(): + print(f' {key}: {value or "unknown"}') + return versions + + def generate_dockerfile(self) -> str: + with open('docker/Dockerfile.amd', 'r') as f: + content = f.read() + content = content.replace('{base_image}', self.args.base_image) + content = content.replace('{base_image_tag}', self.args.base_image_tag) + content = content.replace('{modelscope_branch}', + self.args.modelscope_branch) + content = content.replace('{cur_time}', formatted_time) + return content + + def image(self) -> str: + ubuntu = getattr(self.args, 'amd_ubuntu_version', + None) or self.args.ubuntu_version + rocm = getattr(self.args, 'amd_rocm_version', None) + py_tag = getattr(self.args, 'amd_python_tag', None) or getattr( + self.args, 'python_tag', None) + torch = getattr(self.args, 'amd_torch_version', None) + if not (rocm and py_tag and torch): + raise RuntimeError( + 'AMD image tag requires probed rocm/python/torch versions. ' + f'Got rocm={rocm}, python={py_tag}, torch={torch}') + return (f'{docker_registry}:ubuntu{ubuntu}-rocm{rocm}-{py_tag}-' + f'torch{torch}-{self.args.modelscope_version}-test') + + def _log_base_image_info(self) -> int: + base_image = self.args.base_image + print('=' * 60) + print(f'AMD ROCm base image: {base_image}') + print(f'AMD ROCm base image tag: {self.args.base_image_tag}') + print('=' * 60) + ret = self.run_cmd('docker', 'pull', base_image) + if ret != 0: + return ret + result = self._run_capture( + 'docker', 'image', 'inspect', base_image, + '--format={{.Id}} {{if index .RepoDigests 0}}' + '{{index .RepoDigests 0}}{{else}}local-only{{end}}') + if result.returncode == 0: + print(f'AMD base image resolved: {result.stdout.strip()}') + else: + print(f'AMD base image inspect warning: {result.stderr.strip()}') + + versions = self.probe_base_image_versions(base_image) + if not versions.get('rocm') or not versions.get( + 'python') or not versions.get('torch'): + print('ERROR: failed to probe rocm/python/torch from base image') + return 1 + self.args.amd_rocm_version = versions['rocm'] + self.args.amd_torch_version = versions['torch'] + self.args.amd_python_tag = self._python_tag_from_version( + versions['python']) + if versions.get('ubuntu'): + self.args.amd_ubuntu_version = versions['ubuntu'] + else: + self.args.amd_ubuntu_version = self.args.ubuntu_version + print(f'AMD output image tag will be: {self.image()}') + print('=' * 60) + return 0 + + def build(self) -> int: + ret = self._log_base_image_info() + if ret != 0: + return ret + return self.run_cmd('docker', 'build', '-t', self.image(), '-f', + 'Dockerfile', '.') + + def push(self): + image_name = self.image() + ret = self.run_cmd('docker', 'push', image_name) + if ret != 0: + return ret + ubuntu = self.args.amd_ubuntu_version + rocm = self.args.amd_rocm_version + py_tag = self.args.amd_python_tag + torch = self.args.amd_torch_version + image_tag2 = (f'{docker_registry}:ubuntu{ubuntu}-rocm{rocm}-{py_tag}-' + f'torch{torch}-{self.args.modelscope_version}-' + f'{formatted_time}-test') + ret = self.run_cmd('docker', 'tag', image_name, image_tag2) + if ret != 0: + return ret + print(f'AMD image timestamp tag: {image_tag2}') + return self.run_cmd('docker', 'push', image_tag2) + + class AscendImageBuilder(StableGPUImageBuilder): _DEFAULT_TORCH_VERSION = '2.9.0' @@ -638,7 +1017,8 @@ class AscendImageBuilder(StableGPUImageBuilder): def generate_dockerfile(self) -> str: extra_content = """ -RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy +RUN export PIP_EXTRA_INDEX_URL=https://pypi.org/simple && \ + pip install --no-cache-dir -U icecream soundfile pybind11 py-spy """ with open('docker/Dockerfile.ascend', 'r') as f: content = f.read() @@ -708,6 +1088,12 @@ parser.add_argument('--megatron_branch', type=str, default='v0.15.3') parser.add_argument('--mindspeed_branch', type=str, default='core_r0.15.3') parser.add_argument('--soc_version', type=str, default='ascend910_9391') parser.add_argument('--arch', type=str, choices=['x86', 'arm'], default=None) +parser.add_argument( + '--base_image_tag', + type=str, + default=None, + help='Optional AMD ROCm override tag. Default: auto-resolve newest ' + 'concrete vllm/vllm-openai-rocm release tag from Docker Hub.') parser.add_argument('--dry_run', type=int, default=0) args = parser.parse_args() @@ -719,6 +1105,8 @@ elif args.image_type.lower() == 'stable': builder_cls = [StableCPUImageBuilder, StableGPUImageBuilder] elif args.image_type.lower() == 'ascend': builder_cls = [AscendImageBuilder] +elif args.image_type.lower() == 'amd': + builder_cls = [AmdImageBuilder] elif args.image_type.lower() == 'latest': builder_cls = [LatestGPUImageBuilder] else: diff --git a/docker/scripts/modelscope_env_init.sh b/docker/scripts/modelscope_env_init.sh index 74c2fbe9..d12b2caa 100755 --- a/docker/scripts/modelscope_env_init.sh +++ b/docker/scripts/modelscope_env_init.sh @@ -47,5 +47,4 @@ else fi pip config set global.index-url https://mirrors.cloud.aliyuncs.com/pypi/simple -pip config set global.extra-index-url https://pypi.org/simple pip config set install.trusted-host mirrors.cloud.aliyuncs.com diff --git a/modelscope/hub/file_download.py b/modelscope/hub/file_download.py index d1273f19..759d4753 100644 --- a/modelscope/hub/file_download.py +++ b/modelscope/hub/file_download.py @@ -18,7 +18,8 @@ from typing import Dict, List, Optional, Type import requests # --- Hub file downloads (delegated) --- -from modelscope_hub.compat import dataset_file_download # noqa: E402,F401 +from modelscope_hub.compat.file_download import \ + dataset_file_download as _compat_dataset_file_download from modelscope_hub.compat.file_download import \ model_file_download as _compat_model_file_download from requests.adapters import Retry @@ -31,7 +32,7 @@ from modelscope.hub.constants import (API_FILE_DOWNLOAD_CHUNK_SIZE, from modelscope.utils.logger import get_logger from .callback import ProgressCallback, TqdmCallback from .errors import FileDownloadError -from .utils.utils import get_endpoint +from .utils.utils import find_reusable_legacy_repo_dir, get_endpoint logger = get_logger() @@ -77,6 +78,9 @@ def model_file_download( revision = detail.get('Revision') except Exception: pass + if local_dir is None: + local_dir = find_reusable_legacy_repo_dir( + model_id, repo_type='model', cache_dir=cache_dir) return _compat_model_file_download( model_id, file_path, @@ -91,6 +95,37 @@ def model_file_download( ) +def dataset_file_download( + dataset_id: str, + file_path: str, + *, + cache_dir: str = None, + local_dir: str = None, + revision: str = None, + cookies: dict = None, + token: str = None, + endpoint: str = None, + local_files_only: bool = False, + user_agent=None, +) -> str: + """Download a single dataset file, reusing flat/hub legacy caches when present.""" + if local_dir is None: + local_dir = find_reusable_legacy_repo_dir( + dataset_id, repo_type='dataset', cache_dir=cache_dir) + return _compat_dataset_file_download( + dataset_id, + file_path, + cache_dir=cache_dir, + local_dir=local_dir, + revision=revision, + cookies=cookies, + token=token, + endpoint=endpoint, + local_files_only=local_files_only, + user_agent=user_agent, + ) + + # --- Direct HTTP downloads (retained - non-Hub API) --- diff --git a/modelscope/hub/snapshot_download.py b/modelscope/hub/snapshot_download.py index 1f500ecb..e2738cb2 100644 --- a/modelscope/hub/snapshot_download.py +++ b/modelscope/hub/snapshot_download.py @@ -4,16 +4,58 @@ Delegates to ``modelscope_hub.compat`` while keeping ``revision``, ``cache_dir`` and friends accessible as positional arguments for backward compatibility. """ from __future__ import annotations +import threading from pathlib import Path -from typing import Dict, List, Optional, Union +from typing import TYPE_CHECKING, Dict, List, Optional, Type, Union from modelscope_hub.compat.snapshot_download import \ dataset_snapshot_download as _compat_dataset_snapshot_download from modelscope_hub.compat.snapshot_download import \ snapshot_download as _compat_snapshot_download +from modelscope.hub.utils.utils import find_reusable_legacy_repo_dir +from modelscope.utils.logger import get_logger + +if TYPE_CHECKING: + from .callback import ProgressCallback + +logger = get_logger() + __all__ = ['snapshot_download', 'dataset_snapshot_download'] +# Capability probe: pre-1.38 cache auto-detection lives in modelscope-hub +# (DownloadManager._find_legacy_repo_dir, added in modelscope-hub>=0.1.7). +# Warn once if the installed hub predates it, so an existing legacy cache is +# not silently ignored and re-downloaded into the new layout. +_legacy_cache_capability: Optional[bool] = None +_legacy_cache_lock = threading.Lock() + + +def _warn_if_legacy_cache_detection_unavailable() -> None: + """Warn once when the installed modelscope-hub cannot auto-detect a + pre-1.38 (legacy) cache layout, so downloads don't silently skip an + existing local cache and re-fetch into the new layout. + """ + global _legacy_cache_capability + if _legacy_cache_capability is not None: + return + with _legacy_cache_lock: + if _legacy_cache_capability is not None: + return + try: + from modelscope_hub._download import DownloadManager + _legacy_cache_capability = hasattr(DownloadManager, + '_find_legacy_repo_dir') + except Exception: + _legacy_cache_capability = False + if not _legacy_cache_capability: + logger.warning( + 'The installed modelscope-hub lacks legacy cache ' + 'auto-detection (added in modelscope-hub>=0.1.7). An existing ' + 'pre-1.38 cache will not be reused; files will be downloaded ' + 'into the new cache layout. Upgrade with: ' + "pip install -U 'modelscope-hub>=0.1.8'.") + def snapshot_download( model_id: Optional[str] = None, @@ -30,6 +72,7 @@ def snapshot_download( max_workers: Optional[int] = None, repo_id: Optional[str] = None, repo_type: Optional[str] = None, + progress_callbacks: Optional[List[Type[ProgressCallback]]] = None, token: Optional[str] = None, endpoint: Optional[str] = None, ) -> str: @@ -37,11 +80,20 @@ def snapshot_download( Preserves the legacy positional-argument signature for backward compatibility while delegating to ``modelscope_hub.compat``. + ``progress_callbacks`` is a list of :class:`ProgressCallback` subclasses + (not instances), each instantiated per file to report download progress. """ + _warn_if_legacy_cache_detection_unavailable() + effective_id = repo_id or model_id + effective_type = repo_type or 'model' + cache_dir_str = str(cache_dir) if cache_dir is not None else None + if local_dir is None and effective_id is not None: + local_dir = find_reusable_legacy_repo_dir( + effective_id, repo_type=effective_type, cache_dir=cache_dir_str) return _compat_snapshot_download( model_id=model_id, revision=revision, - cache_dir=str(cache_dir) if cache_dir is not None else None, + cache_dir=cache_dir_str, local_dir=local_dir, allow_file_pattern=allow_file_pattern, ignore_file_pattern=ignore_file_pattern, @@ -56,6 +108,7 @@ def snapshot_download( local_files_only=bool(local_files_only) if local_files_only is not None else False, user_agent=user_agent, + progress_callbacks=progress_callbacks, ) @@ -75,11 +128,16 @@ def dataset_snapshot_download( endpoint: Optional[str] = None, ) -> str: """Download a dataset repo snapshot (legacy positional-arg signature).""" + _warn_if_legacy_cache_detection_unavailable() effective_id = dataset_id or repo_id + cache_dir_str = str(cache_dir) if cache_dir is not None else None + if local_dir is None and effective_id is not None: + local_dir = find_reusable_legacy_repo_dir( + effective_id, repo_type='dataset', cache_dir=cache_dir_str) return _compat_dataset_snapshot_download( dataset_id=effective_id, revision=revision, - cache_dir=str(cache_dir) if cache_dir is not None else None, + cache_dir=cache_dir_str, local_dir=local_dir, allow_file_pattern=allow_file_pattern, ignore_file_pattern=ignore_file_pattern, diff --git a/modelscope/hub/utils/utils.py b/modelscope/hub/utils/utils.py index 7d9a99d4..7b7c042e 100644 --- a/modelscope/hub/utils/utils.py +++ b/modelscope/hub/utils/utils.py @@ -194,6 +194,78 @@ def get_cache_dir(model_id: Optional[str] = None): base_path, model_id + '/') +def _modelscope_hub_cache_root() -> Path: + """Cache root used by ``modelscope_hub`` downloads (not SDK ``.../hub``).""" + env = os.environ.get('MODELSCOPE_CACHE') + if env: + return Path(env).expanduser() + return Path.home() / '.cache' / 'modelscope' + + +def find_reusable_legacy_repo_dir( + repo_id: str, + repo_type: str = 'model', + cache_dir: Optional[Union[str, Path]] = None, +) -> Optional[str]: + """Find old on-disk cache layouts that ``modelscope_hub`` download misses. + + ``modelscope_hub`` reuses ``{cache}/{type}s/{owner}/{safe_name}/`` (dots in + ``name`` replaced by ``___``) and writes to + ``{cache}/{type}s/{owner}--{name}/snapshots/{rev}/``. Older SDKs also + stored repos at: + + - ``{cache}/{owner}/{name}/`` (flat, when ``MODELSCOPE_CACHE`` was set) + - ``{cache}/hub/{owner}/{name}/`` (pre-``models/`` restructuring) + - ``{cache}/{type}s/{owner}/{name}/`` (unsafed name; hub only checks + ``safe_name``) + + Returns a non-empty legacy path only when the layouts hub already handles + are absent, so callers can pass it as ``local_dir`` and avoid re-download. + """ + if not repo_id or '/' not in repo_id: + return None + + base = Path(cache_dir).expanduser() if cache_dir is not None else \ + _modelscope_hub_cache_root() + segment = f'{repo_type}s' if not repo_type.endswith('s') else repo_type + owner, name = repo_id.split('/', 1) + safe_name = name.replace('.', '___') + safe_id = repo_id.replace('/', '--') + + # Layouts already handled by modelscope_hub — do not override. + hub_known = [ + base / segment / safe_id, + base / segment / owner / safe_name, + ] + for path in hub_known: + if _non_empty_dir(path): + return None + + # Layouts hub download does not probe today. + legacy_candidates = [ + base / owner / name, + base / owner / safe_name, + base / 'hub' / owner / name, + base / 'hub' / owner / safe_name, + base / segment / owner / name, + ] + for path in legacy_candidates: + if _non_empty_dir(path): + logger.info('Found legacy cache at %s for %s, reusing.', path, + repo_id) + return str(path) + return None + + +def _non_empty_dir(path: Path) -> bool: + if not path.is_dir(): + return False + try: + return any(path.iterdir()) + except OSError: + return False + + def get_release_datetime(): if MODELSCOPE_SDK_DEBUG in os.environ: rt = int(round(datetime.now().timestamp())) diff --git a/modelscope/utils/automodel_utils.py b/modelscope/utils/automodel_utils.py index 73175dc6..1c46b296 100644 --- a/modelscope/utils/automodel_utils.py +++ b/modelscope/utils/automodel_utils.py @@ -147,12 +147,13 @@ def check_model_from_owner_group(model_dir: str, if group in owner_group: return True # Also check cache path pattern: {cache_root}/{owner}--{model_name}/snapshots/{revision} - # Require exactly "{owner}--{name}" format (2 segments split by --) - # to prevent spoofing via accounts like "iic--hacked" which would - # produce paths like "iic--hacked--evil" and bypass the check. + # Require exactly "{owner}--{name}" with both segments non-empty + # to prevent spoofing via accounts like "iic--hacked" (paths like + # "iic--hacked--evil") or empty names like "iic--". grandparent = os.path.basename(os.path.dirname(parent_dir)) if '--' in grandparent: parts = grandparent.split('--') - if len(parts) == 2 and parts[0] in owner_group: + # Both owner and name must be non-empty; reject "iic--" / "--name". + if len(parts) == 2 and all(parts) and parts[0] in owner_group: return True return False diff --git a/modelscope/utils/hf_util/patcher.py b/modelscope/utils/hf_util/patcher.py index ee26d3d5..7e73ef24 100644 --- a/modelscope/utils/hf_util/patcher.py +++ b/modelscope/utils/hf_util/patcher.py @@ -155,24 +155,82 @@ def _decide_allow_file_pattern(module_name, cls=None): return extra_allow_file_pattern +def _ms_revision(revision): + """Translate an HF revision string into one ModelScope accepts.""" + return 'master' if revision in (None, 'main') else revision + + +def _ms_download_kwargs_from_hf(kwargs, revision=None): + """Map transformers download kwargs onto ``snapshot_download`` arguments. + + Forwards ``local_files_only``, ``cache_dir``, string ``token``, and an + optional revision (normalized via ``_ms_revision``). + """ + download_kwargs = { + 'local_files_only': kwargs.get('local_files_only', False), + } + cache_dir = kwargs.get('cache_dir') + if cache_dir is not None: + download_kwargs['cache_dir'] = cache_dir + token = kwargs.get('token') + if isinstance(token, str): + download_kwargs['token'] = token + if revision is not None: + download_kwargs['revision'] = _ms_revision(revision) + return download_kwargs + + def _get_class_from_dynamic_module(class_reference, *args, **kwargs): """Wrapper that redirects dynamic-module downloads to ModelScope. When a config's ``auto_map`` references another repo, transformers calls ``get_class_from_dynamic_module`` to fetch it. This wrapper ensures that fetch goes through ModelScope instead of HuggingFace. + + Cross-repo ``auto_map`` entries use ``repo_id--module.Class``. After + ``snapshot_download``, the local cache path may itself contain ``--`` + (modelscope_hub 0.1.x layout: ``models/{owner}--{name}/snapshots/...``). + Re-joining that path with ``--`` would make transformers' + ``class_reference.split("--")`` raise ``ValueError``. Instead, pass the + local directory as ``pretrained_model_name_or_path`` and the bare + ``module.Class`` as ``class_reference`` so transformers takes the + ``os.path.isdir`` branch. """ from transformers.dynamic_module_utils import origin_get_class_from_dynamic_module - if 'pretrained_model_name_or_path' in inspect.signature( - origin_get_class_from_dynamic_module).parameters: - pretrained_model_name_or_path = args[0] - if not os.path.exists(pretrained_model_name_or_path): - from modelscope import snapshot_download - args[0] = snapshot_download(pretrained_model_name_or_path) + has_pretrained_arg = ( + 'pretrained_model_name_or_path' + in inspect.signature(origin_get_class_from_dynamic_module).parameters) + # Resolve pretrained_model_name_or_path from kwargs or positional args. + # ``args`` is a tuple; never mutate it in place. + pretrained_in_kwargs = False + pretrained_model_name_or_path = None + if has_pretrained_arg: + if 'pretrained_model_name_or_path' in kwargs: + pretrained_model_name_or_path = kwargs[ + 'pretrained_model_name_or_path'] + pretrained_in_kwargs = True + elif args: + pretrained_model_name_or_path = args[0] + if (pretrained_model_name_or_path is not None + and not os.path.exists(pretrained_model_name_or_path)): + from modelscope import snapshot_download + # Model weights/config: use ``revision`` (not ``code_revision``). + downloaded_path = snapshot_download( + pretrained_model_name_or_path, + **_ms_download_kwargs_from_hf( + kwargs, revision=kwargs.get('revision'))) + if pretrained_in_kwargs: + kwargs['pretrained_model_name_or_path'] = downloaded_path + else: + args = (downloaded_path, ) + args[1:] + pretrained_model_name_or_path = downloaded_path if '--' in class_reference: - repo_id, class_reference = class_reference.split('--') + # Only the first ``--`` is the auto_map delimiter (repo vs module). + repo_id, class_reference = class_reference.split('--', 1) if not os.path.exists(repo_id): - download_kwargs = {} + # Cross-repo code: transformers uses ``code_revision`` for this repo. + download_kwargs = _ms_download_kwargs_from_hf( + kwargs, revision=kwargs.get('code_revision')) extra_allow_file_pattern = _decide_allow_file_pattern( class_reference) if extra_allow_file_pattern is not None: @@ -182,7 +240,18 @@ def _get_class_from_dynamic_module(class_reference, *args, **kwargs): download_kwargs['ignore_file_pattern'] = ignore_file_pattern from modelscope import snapshot_download repo_id = snapshot_download(repo_id, **download_kwargs) - class_reference = repo_id + '--' + class_reference + if has_pretrained_arg: + # Local path + bare class name; do not rejoin with ``--``. + # Keep kwargs/positional form consistent with the original call. + if pretrained_in_kwargs: + kwargs['pretrained_model_name_or_path'] = repo_id + else: + args = (repo_id, ) + args[1:] + else: + # Legacy transformers without pretrained_model_name_or_path. + # Unsafe if repo_id (local cache) contains '--'; modern + # transformers always take the branch above. + class_reference = repo_id + '--' + class_reference return origin_get_class_from_dynamic_module(class_reference, *args, **kwargs) @@ -248,18 +317,16 @@ def _patch_pretrained_class(all_imported_modules, wrap=False): if subfolder: file_filter = f'{subfolder}/*' if not os.path.exists(pretrained_model_name_or_path): - revision = kwargs.pop('revision', None) - if revision is None or revision == 'main': - revision = 'master' + revision = _ms_revision(kwargs.pop('revision', None)) if file_filter is not None: allow_file_pattern = file_filter - local_files_only = kwargs.pop('local_files_only', False) + download_kwargs = _ms_download_kwargs_from_hf( + kwargs, revision=revision) model_dir = snapshot_download( pretrained_model_name_or_path, - revision=revision, - local_files_only=local_files_only, ignore_file_pattern=ignore_file_pattern, - allow_file_pattern=allow_file_pattern) + allow_file_pattern=allow_file_pattern, + **download_kwargs) if subfolder: model_dir = os.path.join(model_dir, subfolder) else: @@ -592,11 +659,6 @@ def _unpatch_kernels(): del kernels_utils._get_hf_api_origin -def _ms_revision(revision): - """Translate an HF revision string into one ModelScope accepts.""" - return 'master' if revision in (None, 'main') else revision - - class _MsKernelApi: """Minimal `HfApi` look-alike that forwards to ModelScope. Only the handful of methods that `kernels` actually calls are implemented. diff --git a/modelscope/utils/plugins.py b/modelscope/utils/plugins.py index fa4cb877..cd8a2135 100644 --- a/modelscope/utils/plugins.py +++ b/modelscope/utils/plugins.py @@ -374,6 +374,30 @@ def create_module_from_files(file_list, file_prefix, module_name): importlib.invalidate_caches() +def _module_name_from_model_dir(model_dir: str) -> str: + """Derive a valid Python package name from a local model directory. + + modelscope_hub 0.1.x returns paths like + ``{cache}/models/{owner}--{name}/snapshots/{revision}``. Using + ``Path(model_dir).stem`` on a revision such as ``v1.0.4`` yields + ``v1.0``, and ``importlib.import_module('v1.0.xxx')`` fails with + ``ModuleNotFoundError: No module named 'v1'`` because dots are package + separators. Prefer the ``{owner}--{name}`` segment (plus revision for + isolation) and sanitize to a valid identifier. + """ + path = Path(model_dir).resolve() + if path.parent.name == 'snapshots': + base = f'{path.parent.parent.name}__{path.name}' + else: + # Use the full directory name, not .stem, so dotted names are kept + # intact before sanitization (stem would turn ``v1.0.4`` into ``v1.0``). + base = path.name + module_name = re.sub(r'[^0-9A-Za-z_]', '_', base) + if not module_name or module_name[0].isdigit(): + module_name = f'm_{module_name}' + return module_name + + def import_module_from_model_dir(model_dir): """ import all the necessary module from a model dir @@ -383,7 +407,6 @@ def import_module_from_model_dir(model_dir): No returns, raise error if failed """ - from pathlib import Path file_scanner = FilesAstScanning() file_scanner.traversal_files(model_dir, include_init=True) file_dirs = file_scanner.file_dirs @@ -395,7 +418,7 @@ def import_module_from_model_dir(model_dir): if BASE_MODULE_DIR not in sys.path: sys.path.append(BASE_MODULE_DIR) - module_name = Path(model_dir).stem + module_name = _module_name_from_model_dir(model_dir) # in order to keep forward compatibility, we add module path to # sys.path so that submodule can be imported directly as before diff --git a/requirements/hub.txt b/requirements/hub.txt index 4b52c05d..0f9ac799 100644 --- a/requirements/hub.txt +++ b/requirements/hub.txt @@ -1,5 +1,5 @@ filelock -modelscope-hub>=0.0.7 +modelscope-hub>=0.1.8 packaging requests>=2.25 setuptools diff --git a/tests/hub/test_download_callback.py b/tests/hub/test_download_callback.py index 598c2ac4..9144a1e2 100644 --- a/tests/hub/test_download_callback.py +++ b/tests/hub/test_download_callback.py @@ -1,5 +1,6 @@ import tempfile import unittest +from unittest import mock from tqdm import tqdm @@ -48,5 +49,36 @@ class ProgressCallbackTest(unittest.TestCase): print(f'model_dir: {model_dir}') +class SnapshotDownloadForwardTest(unittest.TestCase): + """Network-free tests: the shim forwards progress_callbacks to compat.""" + + # ``modelscope.hub.snapshot_download`` the attribute is shadowed by the + # re-exported function, so patch via the fully qualified module string. + _COMPAT_TARGET = \ + 'modelscope.hub.snapshot_download._compat_snapshot_download' + + def test_progress_callbacks_forwarded_to_compat(self): + from modelscope.hub.snapshot_download import snapshot_download + + with mock.patch( + self._COMPAT_TARGET, return_value='/tmp/snapshot') as m: + result = snapshot_download( + 'owner/repo', progress_callbacks=[NewProgressCallback]) + + self.assertEqual(result, '/tmp/snapshot') + _, kwargs = m.call_args + self.assertEqual(kwargs['progress_callbacks'], [NewProgressCallback]) + + def test_progress_callbacks_default_none(self): + from modelscope.hub.snapshot_download import snapshot_download + + with mock.patch( + self._COMPAT_TARGET, return_value='/tmp/snapshot') as m: + snapshot_download('owner/repo') + + _, kwargs = m.call_args + self.assertIsNone(kwargs['progress_callbacks']) + + if __name__ == '__main__': unittest.main() diff --git a/tests/hub/test_legacy_cache_guard.py b/tests/hub/test_legacy_cache_guard.py new file mode 100644 index 00000000..b1cdf66b --- /dev/null +++ b/tests/hub/test_legacy_cache_guard.py @@ -0,0 +1,54 @@ +import importlib +import sys +import unittest +from unittest import mock + + +class LegacyCacheGuardTest(unittest.TestCase): + """Tests for the modelscope-hub capability guard in the shim. + + Network-free: only probes whether the loaded modelscope-hub exposes the + legacy-cache auto-detection capability and warns once when it does not. + """ + + def setUp(self): + # NOTE: ``modelscope.hub.snapshot_download`` the *attribute* is shadowed + # by the re-exported function in ``modelscope.hub.__init__``; use + # importlib to obtain the actual submodule object. + sd = importlib.import_module('modelscope.hub.snapshot_download') + self.sd = sd + # Reset the fire-once probe cache before each test. + sd._legacy_cache_capability = None + + def tearDown(self): + self.sd._legacy_cache_capability = None + + def test_capability_present_no_warning(self): + sd = self.sd + # The real modelscope-hub DownloadManager has _find_legacy_repo_dir. + with mock.patch.object(sd.logger, 'warning') as warn: + sd._warn_if_legacy_cache_detection_unavailable() + self.assertTrue(sd._legacy_cache_capability) + warn.assert_not_called() + + def test_capability_absent_warns_once(self): + sd = self.sd + + class _OldDownloadManager: # lacks _find_legacy_repo_dir + pass + + fake_module = mock.MagicMock() + fake_module.DownloadManager = _OldDownloadManager + + with mock.patch.dict(sys.modules, + {'modelscope_hub._download': fake_module}): + with mock.patch.object(sd.logger, 'warning') as warn: + sd._warn_if_legacy_cache_detection_unavailable() + sd._warn_if_legacy_cache_detection_unavailable() + + self.assertFalse(sd._legacy_cache_capability) + self.assertEqual(warn.call_count, 1) # fire-once + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/hub/test_legacy_cache_reuse.py b/tests/hub/test_legacy_cache_reuse.py new file mode 100644 index 00000000..fc68276a --- /dev/null +++ b/tests/hub/test_legacy_cache_reuse.py @@ -0,0 +1,103 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from modelscope.hub.utils.utils import find_reusable_legacy_repo_dir + + +class LegacyCacheReuseTest(unittest.TestCase): + """Old flat/hub cache layouts should be reusable without re-download.""" + + def setUp(self): + self._tmpdir = tempfile.TemporaryDirectory() + self.cache = Path(self._tmpdir.name) + self.model_id = 'iic/nlp_xlmr_named-entity-recognition_eng-ecommerce-query' + self.owner, self.name = self.model_id.split('/', 1) + + def tearDown(self): + self._tmpdir.cleanup() + + def _touch_model_dir(self, path: Path): + path.mkdir(parents=True, exist_ok=True) + (path / 'configuration.json').write_text('{}', encoding='utf-8') + + def test_finds_flat_legacy_cache(self): + legacy = self.cache / self.owner / self.name + self._touch_model_dir(legacy) + found = find_reusable_legacy_repo_dir( + self.model_id, cache_dir=self.cache) + self.assertEqual(found, str(legacy)) + + def test_finds_hub_legacy_cache(self): + legacy = self.cache / 'hub' / self.owner / self.name + self._touch_model_dir(legacy) + found = find_reusable_legacy_repo_dir( + self.model_id, cache_dir=self.cache) + self.assertEqual(found, str(legacy)) + + def test_reuses_unsafed_models_slash_layout(self): + # Hub only probes safe_name (dots -> ___); unsafed path is reusable. + dotted_id = 'org/model.with.dots' + owner, name = dotted_id.split('/', 1) + slash = self.cache / 'models' / owner / name + self._touch_model_dir(slash) + found = find_reusable_legacy_repo_dir(dotted_id, cache_dir=self.cache) + self.assertEqual(found, str(slash)) + + def test_prefers_hub_known_safe_slash_layout(self): + dotted_id = 'org/model.with.dots' + owner, name = dotted_id.split('/', 1) + safe = self.cache / 'models' / owner / name.replace('.', '___') + flat = self.cache / owner / name + self._touch_model_dir(safe) + self._touch_model_dir(flat) + found = find_reusable_legacy_repo_dir(dotted_id, cache_dir=self.cache) + self.assertIsNone(found) + + def test_prefers_hub_known_owner_dash_layout(self): + modern = self.cache / 'models' / self.model_id.replace('/', '--') + flat = self.cache / self.owner / self.name + self._touch_model_dir(modern) + self._touch_model_dir(flat) + found = find_reusable_legacy_repo_dir( + self.model_id, cache_dir=self.cache) + self.assertIsNone(found) + + def test_empty_legacy_dir_ignored(self): + (self.cache / self.owner / self.name).mkdir(parents=True) + found = find_reusable_legacy_repo_dir( + self.model_id, cache_dir=self.cache) + self.assertIsNone(found) + + def test_uses_modelscope_cache_env(self): + legacy = self.cache / self.owner / self.name + self._touch_model_dir(legacy) + with mock.patch.dict(os.environ, + {'MODELSCOPE_CACHE': str(self.cache)}): + found = find_reusable_legacy_repo_dir(self.model_id) + self.assertEqual(found, str(legacy)) + + def test_default_root_matches_hub_not_sdk_hub_suffix(self): + # Without MODELSCOPE_CACHE, hub uses ~/.cache/modelscope (no /hub). + modern = ( + Path.home() / '.cache' / 'modelscope' / 'models' + / self.model_id.replace('/', '--')) + # Do not create real home dirs; patch the hub root helper instead. + with mock.patch( + 'modelscope.hub.utils.utils._modelscope_hub_cache_root', + return_value=self.cache): + modern_under_test = ( + self.cache / 'models' / self.model_id.replace('/', '--')) + flat = self.cache / self.owner / self.name + self._touch_model_dir(modern_under_test) + self._touch_model_dir(flat) + found = find_reusable_legacy_repo_dir(self.model_id) + self.assertIsNone(found) + self.assertFalse(modern.exists()) # we never touched real home cache + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/utils/test_hf_util.py b/tests/utils/test_hf_util.py index 1dd52b39..e47f8b56 100644 --- a/tests/utils/test_hf_util.py +++ b/tests/utils/test_hf_util.py @@ -249,19 +249,303 @@ class HFUtilTest(unittest.TestCase): f'Expected no weight files in {model_dir}, but found: ' f"{[f for f in files if f.endswith('.safetensors') or f.endswith('.bin')]}" ) - cache_dir = os.path.dirname(model_dir) - cache_dir = os.path.dirname(cache_dir) - model_dir_2 = os.path.join(cache_dir, 'nomic-ai', 'nomic-bert-2048') - if os.path.exists(model_dir_2): - files = os.listdir(model_dir_2) - has_weight_files = any( - f.endswith('.safetensors') or f.endswith('.bin') - for f in files) - self.assertFalse( - has_weight_files, - f'Expected no weight files in {model_dir}, but found: ' - f"{[f for f in files if f.endswith('.safetensors') or f.endswith('.bin')]}" - ) + # modelscope_hub 0.1.x layout uses models/{owner}--{name}/...; + # older layout used {owner}/{name}/. Accept either. + cache_root = model_dir + for _ in range(4): + parent = os.path.dirname(cache_root) + if parent == cache_root: + break + cache_root = parent + candidates = [ + os.path.join(cache_root, 'nomic-ai', 'nomic-bert-2048'), + os.path.join(cache_root, 'models', + 'nomic-ai--nomic-bert-2048'), + ] + for model_dir_2 in candidates: + if not os.path.exists(model_dir_2): + continue + # Walk into snapshots/{rev} if present. + check_dirs = [model_dir_2] + snapshots = os.path.join(model_dir_2, 'snapshots') + if os.path.isdir(snapshots): + check_dirs.extend( + os.path.join(snapshots, d) + for d in os.listdir(snapshots) + if os.path.isdir(os.path.join(snapshots, d))) + for check_dir in check_dirs: + files = os.listdir(check_dir) + has_weight_files = any( + f.endswith('.safetensors') or f.endswith('.bin') + for f in files) + self.assertFalse( + has_weight_files, + f'Expected no weight files in {check_dir}, but found: ' + f"{[f for f in files if f.endswith('.safetensors') or f.endswith('.bin')]}" + ) + + def test_dynamic_module_double_dash_cache_path(self): + """Cross-repo auto_map must survive cache paths that contain '--'. + + modelscope_hub 0.1.x stores repos under ``models/{owner}--{name}/``. + Rejoining that path into ``class_reference`` with ``--`` makes + transformers' ``split("--")`` raise ValueError. + """ + from unittest import mock + + from modelscope.utils.hf_util.patcher import \ + _get_class_from_dynamic_module + + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + local_path = os.path.join(tmp, 'models', 'nomic-ai--nomic-bert-2048', + 'snapshots', 'rev') + os.makedirs(local_path) + pretrained = os.path.join(tmp, 'models', + 'nomic-ai--nomic-embed-text-v1.5', + 'snapshots', 'rev') + os.makedirs(pretrained) + + captured = {} + + def fake_origin(class_reference, pretrained_model_name_or_path, *args, + **kwargs): + # Signature must match transformers so has_pretrained_arg is True. + captured['class_reference'] = class_reference + captured['pretrained'] = pretrained_model_name_or_path + return type('DummyConfig', (), {}) + + class_ref = ('nomic-ai/nomic-bert-2048--' + 'configuration_hf_nomic_bert.NomicBertConfig') + + # create=True: do not permanently leave origin_* on the module + # (would break test_import_not_pollute_dynamic_module). + with mock.patch( + 'transformers.dynamic_module_utils.origin_get_class_from_dynamic_module', + new=fake_origin, + create=True): + with mock.patch( + 'modelscope.snapshot_download', return_value=local_path): + _get_class_from_dynamic_module(class_ref, pretrained) + + # Must pass bare module.Class (no '--') so transformers does not split. + self.assertEqual(captured['class_reference'], + 'configuration_hf_nomic_bert.NomicBertConfig') + # Local cache path (which contains '--') is pretrained_model_name_or_path. + self.assertEqual(captured['pretrained'], local_path) + + def test_dynamic_module_remote_pretrained_tuple_args(self): + """Remote pretrained_model_name_or_path must not mutate args in place. + + ``*args`` is a tuple; ``args[0] = snapshot_download(...)`` raises + TypeError. Rebuild the tuple instead (regression from 9379504f). + """ + from unittest import mock + + from modelscope.utils.hf_util.patcher import \ + _get_class_from_dynamic_module + + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + downloaded = os.path.join(tmp, 'models', 'org--model', 'snapshots', + 'rev') + os.makedirs(downloaded) + + captured = {} + + def fake_origin(class_reference, pretrained_model_name_or_path, *args, + **kwargs): + captured['class_reference'] = class_reference + captured['pretrained'] = pretrained_model_name_or_path + return type('DummyConfig', (), {}) + + # No '--' in class_reference: only the pretrained download branch runs. + remote_id = 'org/model-not-on-disk' + with mock.patch( + 'transformers.dynamic_module_utils.origin_get_class_from_dynamic_module', + new=fake_origin, + create=True): + with mock.patch( + 'modelscope.snapshot_download', + return_value=downloaded) as sd: + # Must not raise TypeError: 'tuple' object does not support + # item assignment. + _get_class_from_dynamic_module('modeling.Foo', remote_id) + + sd.assert_called_once_with(remote_id, local_files_only=False) + self.assertEqual(captured['class_reference'], 'modeling.Foo') + self.assertEqual(captured['pretrained'], downloaded) + + def test_dynamic_module_local_files_only_forwarded(self): + """Download kwargs must be forwarded to both snapshot_download calls. + + Cross-repo auto_map references previously omitted local_files_only / + cache_dir / token / code_revision, so offline and custom-cache loads + still hit the wrong download path for the referenced repo. + """ + from unittest import mock + + from modelscope.utils.hf_util.patcher import \ + _get_class_from_dynamic_module + + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + downloaded = os.path.join(tmp, 'models', 'org--model', 'snapshots', + 'rev') + cross_repo = os.path.join(tmp, 'models', 'org--other', 'snapshots', + 'rev') + os.makedirs(downloaded) + os.makedirs(cross_repo) + + def fake_origin(class_reference, pretrained_model_name_or_path, *args, + **kwargs): + return type('DummyConfig', (), {}) + + remote_id = 'org/model-not-on-disk' + class_ref = 'org/other--configuration_foo.FooConfig' + call_kwargs = [] + cache_dir = os.path.join(tmp, 'custom_cache') + token = 'ms-test-token' + + def fake_download(repo_id, **kwargs): + call_kwargs.append((repo_id, dict(kwargs))) + if repo_id == remote_id: + return downloaded + if repo_id == 'org/other': + return cross_repo + raise AssertionError(f'unexpected download: {repo_id}') + + with mock.patch( + 'transformers.dynamic_module_utils.origin_get_class_from_dynamic_module', + new=fake_origin, + create=True): + with mock.patch( + 'modelscope.snapshot_download', side_effect=fake_download): + _get_class_from_dynamic_module( + class_ref, + pretrained_model_name_or_path=remote_id, + local_files_only=True, + cache_dir=cache_dir, + token=token, + revision='model-rev', + code_revision='code-rev') + + self.assertEqual(len(call_kwargs), 2) + self.assertEqual(call_kwargs[0][0], remote_id) + self.assertEqual( + call_kwargs[0][1], { + 'local_files_only': True, + 'cache_dir': cache_dir, + 'token': token, + 'revision': 'model-rev', + }) + self.assertEqual(call_kwargs[1][0], 'org/other') + self.assertEqual(call_kwargs[1][1]['local_files_only'], True) + self.assertEqual(call_kwargs[1][1]['cache_dir'], cache_dir) + self.assertEqual(call_kwargs[1][1]['token'], token) + self.assertEqual(call_kwargs[1][1]['revision'], 'code-rev') + self.assertIn('ignore_file_pattern', call_kwargs[1][1]) + + def test_ms_download_kwargs_from_hf(self): + """Shared HF→MS download kwargs mapping used by patcher download paths.""" + from modelscope.utils.hf_util.patcher import _ms_download_kwargs_from_hf + + self.assertEqual( + _ms_download_kwargs_from_hf({}), {'local_files_only': False}) + got = _ms_download_kwargs_from_hf( + { + 'local_files_only': True, + 'cache_dir': '/tmp/c', + 'token': 'sekrit', + 'token_ignored': True, + }, + revision='main') + self.assertEqual( + got, { + 'local_files_only': True, + 'cache_dir': '/tmp/c', + 'token': 'sekrit', + 'revision': 'master', + }) + # HF token=True means "default creds"; only string tokens are forwarded. + self.assertNotIn('token', _ms_download_kwargs_from_hf({'token': True})) + + def test_get_model_dir_forwards_cache_dir_and_token(self): + """from_pretrained download path must forward cache_dir and token.""" + from unittest import mock + + from modelscope import AutoConfig + + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + with open(os.path.join(tmp, 'config.json'), 'w') as f: + f.write('{"model_type": "bert", "hidden_size": 8}') + + cache_dir = os.path.join(tmp, 'custom_cache') + token = 'ms-test-token' + with mock.patch( + 'modelscope.snapshot_download', return_value=tmp) as sd: + AutoConfig.from_pretrained( + 'org/model-not-on-disk', + cache_dir=cache_dir, + token=token, + local_files_only=True) + + sd.assert_called_once() + _, kwargs = sd.call_args + self.assertEqual(kwargs.get('cache_dir'), cache_dir) + self.assertEqual(kwargs.get('token'), token) + self.assertTrue(kwargs.get('local_files_only')) + + def test_dynamic_module_pretrained_via_kwargs(self): + """pretrained_model_name_or_path may be passed as a keyword argument.""" + from unittest import mock + + from modelscope.utils.hf_util.patcher import \ + _get_class_from_dynamic_module + + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + downloaded = os.path.join(tmp, 'models', 'org--model', 'snapshots', + 'rev') + cross_repo = os.path.join(tmp, 'models', 'org--other', 'snapshots', + 'rev') + os.makedirs(downloaded) + os.makedirs(cross_repo) + + captured = {} + + def fake_origin(class_reference, pretrained_model_name_or_path, *args, + **kwargs): + captured['class_reference'] = class_reference + captured['pretrained'] = pretrained_model_name_or_path + captured['kwargs'] = kwargs + return type('DummyConfig', (), {}) + + remote_id = 'org/model-not-on-disk' + class_ref = 'org/other--configuration_foo.FooConfig' + + def fake_download(repo_id, **kwargs): + if repo_id == remote_id: + return downloaded + if repo_id == 'org/other': + return cross_repo + raise AssertionError(f'unexpected download: {repo_id}') + + with mock.patch( + 'transformers.dynamic_module_utils.origin_get_class_from_dynamic_module', + new=fake_origin, + create=True): + with mock.patch( + 'modelscope.snapshot_download', side_effect=fake_download): + # Keyword form: must download and not pass duplicate positional. + _get_class_from_dynamic_module( + class_ref, pretrained_model_name_or_path=remote_id) + + self.assertEqual(captured['class_reference'], + 'configuration_foo.FooConfig') + self.assertEqual(captured['pretrained'], cross_repo) + self.assertNotIn('pretrained_model_name_or_path', captured['kwargs']) def test_import_not_pollute_dynamic_module(self): """Importing from modelscope must not globally patch diff --git a/tests/utils/test_owner_group_path_safety.py b/tests/utils/test_owner_group_path_safety.py new file mode 100644 index 00000000..9a0bb5c6 --- /dev/null +++ b/tests/utils/test_owner_group_path_safety.py @@ -0,0 +1,30 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import unittest + +from modelscope.utils.automodel_utils import check_model_from_owner_group + + +class OwnerGroupPathSafetyTest(unittest.TestCase): + """Safety checks for trusted-owner cache path recognition.""" + + def test_empty_name_cache_path_rejected(self): + # modelscope_hub layout: {cache}/{owner}--{name}/snapshots/{rev} + # Empty name ("iic--") must not be treated as a trusted owner path. + self.assertFalse( + check_model_from_owner_group('/cache/iic--/snapshots/v1')) + self.assertFalse( + check_model_from_owner_group('/cache/damo--/snapshots/v1')) + + def test_valid_and_spoof_cache_paths(self): + self.assertTrue( + check_model_from_owner_group('/cache/iic--x/snapshots/v1')) + self.assertFalse( + check_model_from_owner_group('/cache/--iic/snapshots/v1')) + self.assertFalse( + check_model_from_owner_group( + '/cache/iic--hacked--evil/snapshots/v1')) + self.assertTrue(check_model_from_owner_group('/cache/iic/some_model')) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/utils/test_plugin.py b/tests/utils/test_plugin.py index bb01886a..fa0e2872 100644 --- a/tests/utils/test_plugin.py +++ b/tests/utils/test_plugin.py @@ -127,5 +127,40 @@ class PluginTest(unittest.TestCase): self.assertEqual(len(result.items()), len(OFFICIAL_PLUGINS)) +class ModuleNameFromModelDirTest(unittest.TestCase): + """Regression: snapshot revision paths must not become dotted module names.""" + + def test_snapshot_revision_with_dots(self): + from modelscope.utils.plugins import _module_name_from_model_dir + + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + model_dir = os.path.join(tmp, 'models', + 'iic--cv_unet_skin_retouching_torch', + 'snapshots', 'v1.0.4') + os.makedirs(model_dir) + + name = _module_name_from_model_dir(model_dir) + self.assertNotIn('.', name) + self.assertTrue( + name.startswith('iic__cv_unet_skin_retouching_torch__v1_0_4')) + # Old Path(model_dir).stem produced 'v1.0', which importlib treats as + # package 'v1' → ModuleNotFoundError: No module named 'v1'. + self.assertNotEqual(name, 'v1.0') + self.assertFalse(name.startswith('v1')) + + def test_flat_cache_layout_unchanged(self): + from modelscope.utils.plugins import _module_name_from_model_dir + + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + model_dir = os.path.join(tmp, 'iic', 'cv_unet_skin_retouching_torch') + os.makedirs(model_dir) + + self.assertEqual( + _module_name_from_model_dir(model_dir), + 'cv_unet_skin_retouching_torch') + + if __name__ == '__main__': unittest.main()