diff --git a/docker/Dockerfile.ascend b/docker/Dockerfile.ascend index b18eff2f..442a08fe 100644 --- a/docker/Dockerfile.ascend +++ b/docker/Dockerfile.ascend @@ -1,14 +1,13 @@ FROM {base_image} -# Use bash so that `source` and other bash builtins work in all following RUN steps. +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_DEFAULT_TIMEOUT=300 \ + PIP_RETRIES=10 \ + SOC_VERSION={soc_version} + SHELL ["/bin/bash", "-c"] -ENV PIP_DISABLE_PIP_VERSION_CHECK=1 -ENV PIP_DEFAULT_TIMEOUT=300 -ENV PIP_RETRIES=10 -ENV TRANSFORMERS_VERBOSITY=error -ENV TRANSFORMERS_NO_ADVISORY_WARNINGS=1 - +# ---------- System dependencies ---------- RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ find /etc/apt/apt.conf.d -maxdepth 1 -type f | xargs -r grep -l "APT::Update::Post-Invoke\|docker-clean" | xargs -r rm -f && \ apt-get update -y && \ @@ -19,23 +18,62 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ rm -rf /var/lib/apt/lists/* RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \ - pip config set install.trusted-host mirrors.aliyun.com + pip config set install.trusted-host mirrors.aliyun.com && \ + ARCH=$(uname -m) && \ + if [ "$ARCH" = "x86_64" ]; then \ + pip config set global.extra-index-url "https://download.pytorch.org/whl/cpu/"; \ + fi {extra_content} +# ---------- Install vllm + vllm-ascend ---------- +RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ + if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \ + git clone --depth 1 --branch v0.18.0 https://github.com/vllm-project/vllm && \ + git clone --depth 1 --branch v0.18.0 https://github.com/vllm-project/vllm-ascend.git -# Reuse the vllm-ascend base image and only add the extra repos we need. -# --depth 1 keeps the image smaller; branch/tag names work with shallow clone. +RUN ARCH=$(uname -m) && \ + source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ + source /usr/local/Ascend/nnal/atb/set_env.sh && \ + # Install torch & torch_npu & torchvision + pip install torch==2.9.0 torch_npu==2.9.0 torchvision==0.24.0 && \ + # Install vllm + cd vllm && VLLM_TARGET_DEVICE=empty pip install -v -e . && cd .. && \ + # Install vllm-ascend + cd vllm-ascend && pip install -v -e . && cd .. + +# ---------- Clone training-side repositories ---------- RUN git clone --depth 1 --branch v0.15.3 https://github.com/NVIDIA/Megatron-LM.git /Megatron-LM && \ git clone --depth 1 --branch core_r0.15.3 https://gitcode.com/Ascend/MindSpeed.git /MindSpeed && \ GIT_LFS_SKIP_SMUDGE=1 git clone --depth 1 -b {swift_branch} --single-branch https://github.com/modelscope/ms-swift.git /ms-swift && \ git clone --depth 1 https://github.com/modelscope/mcore-bridge.git /mcore-bridge +# ---------- Install training-side repositories ---------- RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \ cd /MindSpeed && pip install --no-cache-dir -e . && \ cd /mcore-bridge && pip install --no-cache-dir -e . && \ - pip cache purge + cd /ms-swift && pip install --no-cache-dir -e . +# ---------- Pin torch to the correct version + torch_npu ---------- +# x86: must force-install the CPU build from pytorch.org/whl/cpu +# aarch64: PyPI only provides the CPU build, so install it directly from the Aliyun mirror +RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ + if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \ + ARCH=$(uname -m) && \ + if [ "$ARCH" = "x86_64" ]; then \ + pip install --no-cache-dir --force-reinstall --no-deps \ + --index-url https://download.pytorch.org/whl/cpu \ + torch==2.9.0 torchvision==0.24.0 torchaudio==2.9.0; \ + else \ + pip install --no-cache-dir --force-reinstall --no-deps \ + torch==2.9.0 torchvision==0.24.0 torchaudio==2.9.0; \ + fi && \ + pip install --no-cache-dir --force-reinstall --no-deps \ + torch_npu==2.9.0 && \ + rm -rf /root/.cache/pip + +# ---------- Remove CUDA-only dependencies pulled in by vllm (they cause missing libtorch_cuda.so errors on NPU) ---------- +RUN pip uninstall -y flashinfer tvm-ffi torch-c-dlpack-ext 2>/dev/null || true ARG INSTALL_MS_DEPS={install_ms_deps} ENV MEGATRON_LM_PATH=/Megatron-LM @@ -68,6 +106,8 @@ fi ARG CUR_TIME={cur_time} RUN echo $CUR_TIME +RUN pip install --no-cache-dir --no-build-isolation OpenCC + RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \ pip install --no-cache-dir -U funasr scikit-learn && \ diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu index e3f2feab..310a8c9c 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.ubuntu @@ -79,7 +79,7 @@ RUN if [ "$IMAGE_TYPE" = "gpu" ]; then \ cd / && rm -fr /tmp/apex && pip cache purge; \ pip install --no-cache-dir "megatron-core==0.17.*" -U; \ elif [ "$IMAGE_TYPE" = "cpu" ]; then \ - pip install --no-cache-dir huggingface-hub transformers peft diffusers -U; \ + pip install --no-cache-dir huggingface-hub "transformers<5.9" peft diffusers -U; \ else \ pip install "transformers<5.0" "tokenizers<0.22" "trl<0.23" "diffusers<0.35" --no-dependencies; \ fi diff --git a/docker/Dockerfile.ubuntu.old b/docker/Dockerfile.ubuntu.old index 6784702e..f8f7f2fc 100644 --- a/docker/Dockerfile.ubuntu.old +++ b/docker/Dockerfile.ubuntu.old @@ -1,5 +1,8 @@ FROM {base_image} +ARG CUR_TIME={cur_time} +RUN echo $CUR_TIME + RUN cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b {modelscope_branch} --single-branch https://github.com/modelscope/modelscope.git && \ cd modelscope && pip install . -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ cd / && rm -fr /tmp/modelscope && pip cache purge; diff --git a/docker/Metax/4.1/Dockerfile.metax b/docker/Metax/4.1/Dockerfile.metax new file mode 100644 index 00000000..07ec8205 --- /dev/null +++ b/docker/Metax/4.1/Dockerfile.metax @@ -0,0 +1,164 @@ +ARG BUILD_BASE_IMAGE=registry.access.redhat.com/ubi9/ubi:9.6 +ARG PYTHON_VERSION=3.12 +ARG UV_EXTRA_INDEX_URL=https://repos.metax-tech.com/r/maca-pypi/simple +ARG UV_TRUSTED_HOST=repos.metax-tech.com + +# may need passing a particular vllm version during build +ARG VLLM_VERSION +ARG MACA_VERSION +ARG CU_BRIDGE_VERSION=${MACA_VERSION} + +#################### BASE BUILD IMAGE #################### +FROM ${BUILD_BASE_IMAGE} AS base +ARG UV_TRUSTED_HOST + +# maca environment variables +ENV MACA_PATH=/opt/maca +ENV MACA_CLANG_PATH=/opt/maca/mxgpu_llvm/bin +ENV CUCC_PATH="${MACA_PATH}/tools/cu-bridge" +ENV CUDA_PATH=/root/cu-bridge/CUDA_DIR +ENV CUCC_CMAKE_ENTRY=2 +ENV PATH="/opt/venv/bin:/root/.local/bin:$PATH" +ENV PATH=/opt/mxdriver/bin:${MACA_PATH}/bin:${MACA_PATH}/mxgpu_llvm/bin:${MACA_PATH}/tools/cu-bridge/tools:${MACA_PATH}/tools/cu-bridge/bin:${PATH} +ENV LD_LIBRARY_PATH=/opt/mxdriver/lib:${MACA_PATH}/lib:${MACA_PATH}/mxgpu_llvm/lib:${MACA_PATH}/ompi/lib:${MACA_PATH}/ucx/lib:${LD_LIBRARY_PATH} + +# uv environment variables +ENV VIRTUAL_ENV=/opt/venv +ENV UV_INDEX_STRATEGY="unsafe-best-match" +ENV UV_HTTP_TIMEOUT=6000 +ENV UV_LINK_MODE=copy +ARG UV_EXTRA_INDEX_URL +ENV UV_EXTRA_INDEX_URL=${UV_EXTRA_INDEX_URL} +ARG UV_INDEX_URL +ENV UV_INDEX_URL=https://mirrors.aliyun.com/pypi/simple +ENV UV_TRUSTED_INDEX_HOST=mirrors.aliyun.com +ENV UV_OVERRIDE=/workspace/override.txt + +# vllm compile option +ENV VLLM_INSTALL_PUNICA_KERNELS=1 + +# AI version arguments +ARG PYTHON_VERSION +ARG VLLM_VERSION +ARG VLLM_METAX_VERSION +ARG MACA_VERSION +ARG MEGATRON_VERSION +ARG SWIFT_VERSION +ARG CU_BRIDGE_VERSION +ARG TE_VERSION + +WORKDIR /workspace +COPY override.txt /workspace/override.txt +COPY requirements_extra.txt /workspace/requirements_extra.txt + +RUN printf "[metax-centos]\n\ +name=Maca Driver Yum Repository\n\ +baseurl=https://repos.metax-tech.com/r/metax-driver-centos-$(uname -m)/\n\ +enabled=1\n\ +gpgcheck=0" > /etc/yum.repos.d/metax-driver-centos.repo + +RUN dnf -y install python3-pip hostname && \ + dnf clean all + +RUN python3 -m pip install uv -i $UV_INDEX_URL --trusted-host ${UV_TRUSTED_INDEX_HOST} && \ + uv venv /opt/venv --python=${PYTHON_VERSION} + +RUN python3 --version && \ + uv self version + +RUN yum makecache && yum install -y \ + unzip vim git openblas-devel make cmake \ + ninja-build gcc g++ procps-ng \ + libibverbs librdmacm libibumad \ + && yum clean all + +RUN git clone --depth 1 --branch ${SWIFT_VERSION} https://github.com/modelscope/ms-swift.git +RUN git clone --depth 1 --branch ${VLLM_METAX_VERSION} https://github.com/MetaX-MACA/vLLM-metax.git +RUN git clone --depth 1 --branch ${VLLM_VERSION} https://github.com/vllm-project/vllm.git +RUN git clone --depth 1 --branch ${MEGATRON_VERSION} https://github.com/NVIDIA/Megatron-LM.git + +# Step 1: install MACA SDK, Metax-Driver and cu-bridge +# Metax-Driver mainly contains vbios and kmd files, which are not needed in a container. +# Here we keep the mx-smi management tool. Kernel version mismatch errors are ignored. +RUN yum makecache && \ + yum install -y metax-driver-${MACA_VERSION}* mxgvm && \ + yum clean all && rm -rf /var/cache/yum /tmp/* + +RUN printf "[maca-sdk]\n\ +name=Maca Sdk Yum Repository\n\ +baseurl=https://repos.metax-tech.com/r/maca-sdk-rpm-$(uname -m)/\n\ +enabled=1\n\ +gpgcheck=0" > /etc/yum.repos.d/maca-sdk-rpm.repo + +RUN yum makecache && \ + yum install -y maca_sdk-${MACA_VERSION}* && \ + yum clean all && rm -rf /var/cache/yum /tmp/* + +RUN cd /tmp/ && \ + export MACA_PATH=/opt/maca && \ + curl -o ${CU_BRIDGE_VERSION}.zip -LsSf https://gitee.com/metax-maca/cu-bridge/repository/archive/${CU_BRIDGE_VERSION}.zip && \ + unzip ${CU_BRIDGE_VERSION}.zip && \ + mv cu-bridge-${CU_BRIDGE_VERSION} cu-bridge && \ + chmod 755 cu-bridge -Rf && \ + cd cu-bridge && \ + mkdir build && cd build && \ + cmake -DCMAKE_INSTALL_PREFIX=/opt/maca/tools/cu-bridge ../ && \ + make && make install + +# Step 2: trim unused MACA packages and install build prerequisites +RUN cd vLLM-metax && \ + uv pip install -r requirements/build.txt && \ + uv pip install build + +RUN yum makecache && yum install -y \ + gcc \ + binutils \ + procps-ng \ + libibverbs \ + librdmacm \ + libibumad \ + openblas \ + numactl-libs \ + && yum clean all && rm -rf /var/cache/yum /tmp/* + +# Step 3: install Metax python requirements +RUN cd vLLM-metax && \ + UV_HTTP_TIMEOUT=960 uv pip install -r requirements/maca.txt --trusted-host ${UV_TRUSTED_HOST} + +# Step 4: build vLLM with empty device to avoid CUDA dependency +RUN cd vllm && \ + python3 use_existing_torch.py && \ + uv pip install -r requirements/build.txt + +RUN cd vllm && \ + VLLM_TARGET_DEVICE=empty uv pip install -v . --no-build-isolation + +# Step 5: build vLLM-metax +RUN cd vLLM-metax && \ + uv pip install -r requirements/build.txt && \ + python3 -m build -w -n && \ + uv pip install dist/*.whl + +# Step 6: install Megatron-LM +RUN sed -i 's/nvcc/cucc/g' /workspace/Megatron-LM/megatron/legacy/fused_kernels/__init__.py && \ + cd Megatron-LM && \ + uv pip install . + +# Step 7: install transformer-engine +RUN uv pip install transformer_engine==${TE_VERSION} -i https://repos.metax-tech.com/r/maca-pypi/simple --trusted-host ${UV_TRUSTED_HOST} + +# Step 8: patch and install ms-swift v4.1.0 with Megatron extra dependencies +RUN sed -i '0,/^\(from \|import \)/{s//import vllm_metax.patch\n&/}' ms-swift/swift/__init__.py && \ + cd ms-swift && \ + uv pip install '.[megatron]' + +# Step 9: install optional runtime dependencies used by swift 4.1.0 +RUN uv pip install deepspeed -i https://repos.metax-tech.com/r/maca-pypi/simple --trusted-host ${UV_TRUSTED_HOST} +RUN uv pip install pip +RUN uv pip install -r requirements_extra.txt +RUN ln -sf ${CUDA_PATH}/bin/nvcc ${CUDA_PATH}/bin/cucc + +# vllm installation may bring in incompatible CUDA-only wheels. Remove them here. +RUN uv pip uninstall flashinfer-python cupy-cuda12x flash-linear-attention fla-core + +#################### FINAL IMAGE #################### diff --git a/docker/Metax/4.1/Dockerfile.with_metax_image b/docker/Metax/4.1/Dockerfile.with_metax_image new file mode 100644 index 00000000..296f5668 --- /dev/null +++ b/docker/Metax/4.1/Dockerfile.with_metax_image @@ -0,0 +1,73 @@ +ARG BUILD_BASE_IMAGE=mx-devops-acr-cn-shanghai.cr.volces.com/opensource/public-ai-release/maca/ms-swift:4.0.4-maca.ai3.5.3.5-torch2.8-py312-ubuntu22.04-amd64 +ARG PYTHON_VERSION=3.12 + +FROM ${BUILD_BASE_IMAGE} AS base + +# NOTE: +# This fast-build path inherits Python/Torch/TE from a prebuilt Metax release image. +# We keep the verified base image tag here instead of guessing a newer one. +# As a result, this path may lag behind the Megatron-SWIFT Quick Start recommendations. + +# may need passing a particular vllm version during build +ARG VLLM_VERSION +ARG VLLM_METAX_VERSION +ARG MEGATRON_VERSION +ARG SWIFT_VERSION + +ENV MACA_PATH=/opt/maca +ENV CUCC_CMAKE_ENTRY=2 +ENV CUDA_PATH=/root/cu-bridge/CUDA_DIR +ENV CUCC_PATH=${MACA_PATH}/tools/cu-bridge +ENV PATH=/opt/conda/bin:/opt/conda/condabin:${CUDA_PATH}/bin:${CUCC_PATH}/tools:${CUCC_PATH}/bin:${MACA_PATH}/bin:${PATH} +ENV LD_LIBRARY_PATH=${CUDA_PATH}/lib64:${MACA_PATH}/lib:${MACA_PATH}/mxgpu_llvm/lib:${LD_LIBRARY_PATH} + +WORKDIR /workspace +COPY requirements_extra.txt /workspace/requirements_extra.txt + +RUN echo $PATH +RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* + +# Initialize cu-bridge if it is not already prepared in the base image. +RUN if [ ! -d /root/cu-bridge ]; then \ + ${MACA_PATH}/tools/cu-bridge/tools/pre_make; \ + fi + +# Clone all GitHub sources while the external proxy is enabled. +RUN rm -rf /workspace/ms-swift /workspace/vLLM-metax /workspace/vllm /workspace/Megatron-LM +RUN git clone --depth 1 --branch ${SWIFT_VERSION} https://github.com/modelscope/ms-swift.git +RUN git clone --depth 1 --branch ${VLLM_METAX_VERSION} https://github.com/MetaX-MACA/vLLM-metax.git +RUN git clone --depth 1 --branch ${VLLM_VERSION} https://github.com/vllm-project/vllm.git +RUN git clone --depth 1 --branch ${MEGATRON_VERSION} https://github.com/NVIDIA/Megatron-LM.git + +# install cmake +RUN pip install cmake ninja + +# Step 1: build original vLLM for torch setup +RUN cd vllm && \ + python3 use_existing_torch.py && \ + pip install -r requirements/build.txt + +# Step 2: build vLLM with empty device to avoid CUDA dependency +RUN cd vllm && \ + VLLM_TARGET_DEVICE=empty pip install -v . --no-build-isolation + +# Step 3: build vLLM-metax +RUN cd vLLM-metax && \ + python3 use_existing_metax.py && \ + pip install -r requirements/build.txt && \ + python3 -m build -w -n && \ + pip install dist/*.whl + +# Step 4: patch and install Megatron-LM +RUN sed -i 's/nvcc/cucc/g' /workspace/Megatron-LM/megatron/legacy/fused_kernels/__init__.py && \ + cd /workspace/Megatron-LM && \ + pip install . + +# Step 5: patch and install ms-swift v4.1.0 with its Megatron extra +RUN sed -i '0,/^\(from \|import \)/{s//import vllm_metax.patch\n&/}' ms-swift/swift/__init__.py && \ + cd ms-swift && \ + pip install "transformers<5.4.0" && \ + pip install '.[megatron]' && \ + pip install -r /workspace/requirements_extra.txt + +CMD ["bash"] diff --git a/docker/Metax/4.1/build.sh b/docker/Metax/4.1/build.sh new file mode 100644 index 00000000..6c112b53 --- /dev/null +++ b/docker/Metax/4.1/build.sh @@ -0,0 +1,13 @@ +docker build \ + --network host \ + -f Dockerfile.metax \ + -t swift:v4.1.0 \ + --build-arg VLLM_VERSION=v0.17.1 \ + --build-arg VLLM_METAX_VERSION=v0.17.0 \ + --build-arg MACA_VERSION=3.5.3 \ + --build-arg MEGATRON_VERSION=core_v0.16.0 \ + --build-arg SWIFT_VERSION=v4.1.0 \ + --build-arg TE_VERSION=2.8.0 \ + --build-arg CU_BRIDGE_VERSION=3.5.3 \ + --no-cache \ + . diff --git a/docker/Metax/4.1/build_from_metax_image.sh b/docker/Metax/4.1/build_from_metax_image.sh new file mode 100644 index 00000000..3c2fb454 --- /dev/null +++ b/docker/Metax/4.1/build_from_metax_image.sh @@ -0,0 +1,10 @@ +docker build \ + --network host \ + -f Dockerfile.with_metax_image \ + -t swift:v4.1.0 \ + --build-arg VLLM_VERSION=v0.17.1 \ + --build-arg VLLM_METAX_VERSION=v0.17.0 \ + --build-arg MEGATRON_VERSION=core_v0.16.0 \ + --build-arg SWIFT_VERSION=v4.1.0 \ + --no-cache \ + . diff --git a/docker/Metax/4.1/override.txt b/docker/Metax/4.1/override.txt new file mode 100644 index 00000000..4a3f3b38 --- /dev/null +++ b/docker/Metax/4.1/override.txt @@ -0,0 +1,5 @@ +setuptools>=77.0.3,<80 +datasets>=3.0,<4.0 +flash-linear-attention +mcoplib +transformers<5.4.0 diff --git a/docker/Metax/4.1/requirements_extra.txt b/docker/Metax/4.1/requirements_extra.txt new file mode 100644 index 00000000..3480e1d9 --- /dev/null +++ b/docker/Metax/4.1/requirements_extra.txt @@ -0,0 +1,14 @@ +decord +diffusers==0.35.2 +evalscope>=1.0.0 +evalscope[opencompass] +evalscope[vlmeval] +keye_vl_utils>=1.5.2 +librosa +mpi4py +optimum==1.27.0 +pytorchvideo +qwen_omni_utils>=0.0.9 +qwen_vl_utils==0.0.14 +soundfile +timm diff --git a/docker/Metax/4.1/swift_building_instructions.md b/docker/Metax/4.1/swift_building_instructions.md new file mode 100644 index 00000000..c7a33091 --- /dev/null +++ b/docker/Metax/4.1/swift_building_instructions.md @@ -0,0 +1,52 @@ +# 1. Build swift 4.1 image from a UBI9 base image + Full build from a minimal base image, using a venv virtual environment. + +## 1.1. Build + ``` bash + bash build.sh + ``` + +## 1.2. Run a container + ``` bash + docker run -d -it --net=host --uts=host --ipc=host --privileged=true --group-add video \ + --shm-size 100gb --ulimit memlock=-1 \ + --security-opt seccomp=unconfined --security-opt apparmor=unconfined \ + --device=/dev/dri --device=/dev/mxcd \ + --name base_image \ + ${IMAGE_ID} bash + ``` + +## 1.3. Activate the venv environment + ``` bash + source /opt/venv/bin/activate + ``` + +## 1.4. Run swift examples + ``` bash + cd /workspace/ms-swift + bash examples/train/full/train.sh + ``` + +# 2. Build swift 4.1 image from a Metax release image + Faster build based on the pre-built Metax release image. + +## 2.1. Build + ``` bash + bash build_from_metax_image.sh + ``` + +## 2.2. Run a container + ``` bash + docker run -d -it --net=host --uts=host --ipc=host --privileged=true --group-add video \ + --shm-size 100gb --ulimit memlock=-1 \ + --security-opt seccomp=unconfined --security-opt apparmor=unconfined \ + --device=/dev/dri --device=/dev/mxcd \ + --name base_image \ + ${IMAGE_ID} bash + ``` + +## 2.3. Run swift examples + ``` bash + cd /workspace/ms-swift + bash examples/train/full/train.sh + ``` diff --git a/docker/build_image.py b/docker/build_image.py index 246f9e9f..c70824e8 100644 --- a/docker/build_image.py +++ b/docker/build_image.py @@ -1,6 +1,8 @@ import argparse import os +import platform import subprocess +from copy import copy from datetime import datetime from typing import Any @@ -24,9 +26,9 @@ class Builder: # A mirrored image of nvidia/cuda:12.4.0-devel-ubuntu22.04 args.base_image = 'nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04' if not args.torch_version: - args.torch_version = '2.9.1' - args.torchaudio_version = '2.9.1' - args.torchvision_version = '0.24.1' + args.torch_version = '2.10.0' + args.torchaudio_version = '2.10.0' + args.torchvision_version = '0.25.0' if not args.optimum_version: args.optimum_version = '2.0.0' if not args.tf_version: @@ -34,7 +36,7 @@ class Builder: if not args.cuda_version: args.cuda_version = '12.8.1' if not args.vllm_version: - args.vllm_version = '0.15.1' + args.vllm_version = '0.19.1' if not args.lmdeploy_version: args.lmdeploy_version = '0.11.0' if not args.autogptq_version: @@ -139,6 +141,7 @@ class OldCPUImageBuilder(Builder): content = content.replace('{base_image}', old_cpu_image) content = content.replace('{modelscope_branch}', self.args.modelscope_branch) + content = content.replace('{cur_time}', formatted_time) return content def image(self) -> str: @@ -196,6 +199,7 @@ class OldGPUImageBuilder(Builder): content = content.replace('{base_image}', old_gpu_image) content = content.replace('{modelscope_branch}', self.args.modelscope_branch) + content = content.replace('{cur_time}', formatted_time) return content def image(self) -> str: @@ -338,6 +342,15 @@ class StableCPUImageBuilder(Builder): class StableGPUImageBuilder(Builder): """Dependencies will be stable versions""" + def init_args(self, args: Any) -> Any: + if not args.torch_version: + args.torch_version = '2.10.0' + args.torchaudio_version = '2.10.0' + args.torchvision_version = '0.25.0' + if not args.vllm_version: + args.vllm_version = '0.19.1' + return super().init_args(args) + def generate_dockerfile(self) -> str: meta_file = './docker/install.sh' with open('docker/Dockerfile.extra_install', 'r') as f: @@ -469,10 +482,46 @@ RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy class AscendImageBuilder(StableGPUImageBuilder): + @staticmethod + def _normalize_arch(arch: str = None) -> str: + arch = arch or platform.machine() + arch = arch.lower() + arch_mapping = { + 'x86': 'x86', + 'x86_64': 'x86', + 'amd64': 'x86', + 'arm': 'arm', + 'aarch64': 'arm', + 'arm64': 'arm', + } + if arch not in arch_mapping: + raise ValueError(f'Unsupported architecture: {arch}. ' + 'Please pass --arch x86 or --arch arm.') + return arch_mapping[arch] + + @staticmethod + def _get_atlas_hardware(soc_version: str) -> str: + soc_version = soc_version.lower() + atlas_mapping = { + 'ascend910b1': 'A2', + 'ascend910_9391': 'A3', + 'ascend310p1': '300I', + } + if soc_version.startswith('ascend950'): + return 'A5' + if soc_version not in atlas_mapping: + raise ValueError( + f'Unsupported soc_version: {soc_version}. ' + 'Supported values are ascend910b1, ascend910_9391, ' + 'ascend310p1, and values starting with ascend950.') + return atlas_mapping[soc_version] + def init_args(self, args) -> Any: if not args.base_image: # Reuse the prebuilt vllm-ascend image to avoid rebuilding its stack. - args.base_image = 'quay.io/ascend/vllm-ascend:v0.14.0rc1-a3' + args.base_image = 'quay.io/ascend/cann:8.5.1-a3-ubuntu22.04-py3.11' + args.arch = self._normalize_arch(args.arch) + args.atlas_hardware = self._get_atlas_hardware(args.soc_version) return super().init_args(args) def generate_dockerfile(self) -> str: @@ -482,6 +531,7 @@ RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy with open('docker/Dockerfile.ascend', 'r') as f: content = f.read() content = content.replace('{base_image}', self.args.base_image) + content = content.replace('{soc_version}', self.args.soc_version) content = content.replace('{extra_content}', extra_content) content = content.replace('{cur_time}', formatted_time) content = content.replace('{install_ms_deps}', 'False') @@ -492,8 +542,9 @@ RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy def image(self) -> str: return ( - f'{docker_registry}:{self.args.base_image.split(":")[-1]}-torch2.7.1' - f'-{self.args.modelscope_version}-ascend-test') + f'{docker_registry}:{self.args.swift_branch}-' + f'{self.args.atlas_hardware}-{self.args.python_tag}-{self.args.arch}' + ) def push(self): return 0 @@ -518,6 +569,8 @@ parser.add_argument('--autogptq_version', type=str, default=None) parser.add_argument('--modelscope_branch', type=str, default='master') parser.add_argument('--modelscope_version', type=str, default='9.99.0') parser.add_argument('--swift_branch', type=str, default='main') +parser.add_argument('--soc_version', type=str, default='ascend910_9391') +parser.add_argument('--arch', type=str, choices=['x86', 'arm'], default=None) parser.add_argument('--dry_run', type=int, default=0) args = parser.parse_args() @@ -535,4 +588,5 @@ else: raise ValueError(f'Unsupported image_type: {args.image_type}') for builder in builder_cls: + args = copy(args) builder(args, args.dry_run)() diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 4e700ad7..2de69a44 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -2316,6 +2316,18 @@ class HubApi: commit_message=commit_message, ) + # Guard: skip sending empty commits (no effective file changes) + if not payload['actions']: + logger.info( + 'Commit skipped: no effective actions in payload ' + '(all files already exist).') + return CommitInfo( + commit_url='', + commit_message=commit_message, + commit_description=commit_description or '', + oid='no-op', + ) + response = self.session.post( url, headers=self.builder_headers(self.headers), @@ -2753,8 +2765,9 @@ class HubApi: ) commit_description = commit_description or 'Uploading files' - # Exclude internal cache/checkpoint files from upload - _internal_ignore = [UPLOAD_HASH_CACHE_FILE, _LEGACY_PROGRESS_FILE] + # Exclude internal cache/checkpoint files from upload at any directory depth + _internal_files = [UPLOAD_HASH_CACHE_FILE, _LEGACY_PROGRESS_FILE] + _internal_ignore = [p for f in _internal_files for p in (f, f'*/{f}')] if ignore_patterns is None: ignore_patterns = _internal_ignore elif isinstance(ignore_patterns, str): @@ -2957,18 +2970,27 @@ class HubApi: except Exception as e: logger.error( f'Batch {batch_idx + 1}/{num_batches} commit failed: {e}') - for r in results: - tracker.mark_failed( - r['file_path_in_repo'], r['file_mtime'], - r['file_size_on_disk'], - error_type='commit_failed') - # Recover uploaded files to retry queue - for r in results: - total_failed_files.append( - ((r['file_path_in_repo'], r['file_path']), e)) - logger.warning( - f'Batch {batch_idx + 1}/{num_batches}: ' - f'{len(results)} uploaded file(s) recovered to retry queue.') + category = classify_error(e) + if not category.is_retryable: + # Permanent error: mark files as failed, do not retry + for r in results: + tracker.mark_failed( + r['file_path_in_repo'], r['file_mtime'], + r['file_size_on_disk'], + error_type='commit_' + category.value) + logger.error( + f'Batch {batch_idx + 1}/{num_batches}: ' + f'permanent failure ({category.value}), ' + f'{len(results)} file(s) will not be retried.') + else: + # Transient error: recover to retry queue + for r in results: + total_failed_files.append( + ((r['file_path_in_repo'], r['file_path']), e)) + logger.warning( + f'Batch {batch_idx + 1}/{num_batches}: ' + f'{len(results)} file(s) recovered to retry queue ' + f'(error_category={category.value}).') finally: tracker.save() @@ -3030,10 +3052,19 @@ class HubApi: except Exception as e: logger.error( f' Retry round {retry_round + 1} commit failed: {e}') - for result in retry_successes: - retry_failures.append( - ((result['file_path_in_repo'], - result.get('file_path', '')), e)) + category = classify_error(e) + if not category.is_retryable: + for result in retry_successes: + tracker.mark_failed( + result['file_path_in_repo'], + result['file_mtime'], + result['file_size_on_disk'], + error_type='commit_' + category.value) + else: + for result in retry_successes: + retry_failures.append( + ((result['file_path_in_repo'], + result.get('file_path', '')), e)) total_failed_files = retry_failures # Final tracker save @@ -3253,7 +3284,7 @@ class HubApi: repo_type=repo_type, revision=revision) commit_infos.append(commit_info) - # Mark committed + # Mark committed only after successful commit self._track_committed_batch(tracker, batch) logger.info( f'[ReAct] {round_name}: ' @@ -3261,11 +3292,18 @@ class HubApi: except Exception as e: logger.error( f'[ReAct] {round_name} commit failed: {e}') - # Recover uploaded files back to failures - for r in batch: - round_failures.append( - ((r['file_path_in_repo'], - r['file_path']), e)) + category = classify_error(e) + if not category.is_retryable: + for r in batch: + tracker.mark_failed( + r['file_path_in_repo'], r['file_mtime'], + r['file_size_on_disk'], + error_type='commit_' + category.value) + else: + for r in batch: + round_failures.append( + ((r['file_path_in_repo'], + r['file_path']), e)) # OBSERVE: classify new failures, enforce per-file retry limit new_retryable = [] diff --git a/modelscope/models/base/base_model.py b/modelscope/models/base/base_model.py index e19227f0..4a1b6494 100644 --- a/modelscope/models/base/base_model.py +++ b/modelscope/models/base/base_model.py @@ -14,7 +14,8 @@ from modelscope.utils.config import Config, ConfigDict from modelscope.utils.constant import DEFAULT_MODEL_REVISION, Invoke, ModelFile from modelscope.utils.device import verify_device from modelscope.utils.logger import get_logger -from modelscope.utils.plugins import (register_modelhub_repo, +from modelscope.utils.plugins import (filter_plugin_in_whitelist, + register_modelhub_repo, register_plugins_repo) logger = get_logger() @@ -31,7 +32,9 @@ class Model(ABC): device_name = kwargs.get('device', 'gpu') verify_device(device_name) self._device_name = device_name - self.trust_remote_code = kwargs.get('trust_remote_code', False) + self.trust_remote_code = kwargs.get( + 'trust_remote_code', + False) or check_model_from_owner_group(model_dir) def __call__(self, *args, **kwargs) -> Dict[str, Any]: return self.postprocess(self.forward(*args, **kwargs)) @@ -136,7 +139,8 @@ class Model(ABC): kwargs.pop(Invoke.KEY) else: invoked_by = Invoke.PRETRAINED - + _model_trusted = check_model_from_owner_group(model_name_or_path) + trust_remote_code = trust_remote_code or _model_trusted ignore_file_pattern = kwargs.pop('ignore_file_pattern', None) if osp.exists(model_name_or_path): local_model_dir = model_name_or_path @@ -190,7 +194,7 @@ class Model(ABC): # Security check: Only allow execution of remote code or plugins if trust_remote_code is True plugins = cfg.safe_get('plugins') - if plugins and not trust_remote_code: + if filter_plugin_in_whitelist(plugins) and not trust_remote_code: raise RuntimeError( 'Detected plugins field in the model configuration file, but ' 'trust_remote_code=True was not explicitly set.\n' @@ -203,7 +207,7 @@ class Model(ABC): 'Please make sure that you can trust the external codes.') register_modelhub_repo(local_model_dir, allow_remote=trust_remote_code) default_args = {} - if trust_remote_code: + if trust_remote_code and not _model_trusted: default_args = {'trust_remote_code': trust_remote_code} register_plugins_repo(plugins) for k, v in kwargs.items(): diff --git a/modelscope/models/cv/anydoor/anydoor_model.py b/modelscope/models/cv/anydoor/anydoor_model.py index ff9216c1..ef618f0d 100644 --- a/modelscope/models/cv/anydoor/anydoor_model.py +++ b/modelscope/models/cv/anydoor/anydoor_model.py @@ -343,7 +343,7 @@ class ControlLDM(LatentDiffusion, Model): self.only_mid_control = only_mid_control self.control_scales = [1.0] * 13 self.trust_remote_code = kwargs.get('trust_remote_code', False) - self.check_trust_remote_code() + self.check_trust_remote_code(self.model_dir) @torch.no_grad() def get_input(self, batch, k, bs=None, *args, **kwargs): diff --git a/modelscope/models/cv/image_view_transform/image_view_transform_infer.py b/modelscope/models/cv/image_view_transform/image_view_transform_infer.py index e7cac98e..0281f5fa 100644 --- a/modelscope/models/cv/image_view_transform/image_view_transform_infer.py +++ b/modelscope/models/cv/image_view_transform/image_view_transform_infer.py @@ -68,7 +68,7 @@ class ImageViewTransform(TorchModel): self.model = None self.model = load_model_from_config( self.model, config, ckpt, device=self.device) - self.check_trust_remote_code() + self.check_trust_remote_code(model_dir=model_dir) def forward(self, model_path, x, y): pred_results = _infer(self.model, model_path, x, y, self.device) diff --git a/modelscope/models/cv/tinynas_detection/detector.py b/modelscope/models/cv/tinynas_detection/detector.py index dfab0889..e6e94f6b 100644 --- a/modelscope/models/cv/tinynas_detection/detector.py +++ b/modelscope/models/cv/tinynas_detection/detector.py @@ -29,7 +29,7 @@ class SingleStageDetector(TorchModel): init model by cfg """ super().__init__(model_dir, *args, **kwargs) - self.check_trust_remote_code() + self.check_trust_remote_code(model_dir=model_dir) config_path = osp.join(model_dir, self.config_name) config = parse_config(config_path) self.cfg = config diff --git a/modelscope/models/cv/video_depth_estimation/dro_model.py b/modelscope/models/cv/video_depth_estimation/dro_model.py index f39ee423..b32076f3 100644 --- a/modelscope/models/cv/video_depth_estimation/dro_model.py +++ b/modelscope/models/cv/video_depth_estimation/dro_model.py @@ -33,7 +33,7 @@ class DROEstimation(TorchModel): def __init__(self, model_dir: str, **kwargs): """str -- model file root.""" super().__init__(model_dir, **kwargs) - self.check_trust_remote_code() + self.check_trust_remote_code(model_dir=model_dir) model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) diff --git a/modelscope/msdatasets/ms_dataset.py b/modelscope/msdatasets/ms_dataset.py index a18532e0..55ca949f 100644 --- a/modelscope/msdatasets/ms_dataset.py +++ b/modelscope/msdatasets/ms_dataset.py @@ -246,7 +246,7 @@ class MsDataset: 'you can trust the external codes.') # Raise csv field size limit to avoid errors with large cells - if config_kwargs.get('engine') == 'python': + if config_kwargs.pop('engine', None) == 'python': import csv as csv_module import sys try: diff --git a/modelscope/msdatasets/utils/_module_factories.py b/modelscope/msdatasets/utils/_module_factories.py index 861048ff..0dc12f27 100644 --- a/modelscope/msdatasets/utils/_module_factories.py +++ b/modelscope/msdatasets/utils/_module_factories.py @@ -9,12 +9,12 @@ by :func:`~hf_datasets_util.load_dataset_with_ctx`. import importlib import inspect import os +import re from functools import partial from pathlib import Path -from typing import Dict, List, Optional, Sequence, Tuple, Union +from typing import Dict, List, Optional, Tuple -from datasets import (BuilderConfig, DownloadConfig, DownloadMode, Features, - Version, config, data_files) +from datasets import (BuilderConfig, DownloadConfig, config) from datasets.data_files import ( FILES_TO_IGNORE, DataFilesDict, EmptyDatasetError, _get_data_files_patterns, _is_inside_unrequested_special_dir, @@ -24,17 +24,16 @@ from datasets.download.streaming_download_manager import ( _prepare_path_and_storage_options, xbasename, xjoin) from datasets.exceptions import DataFilesNotFoundError from datasets.info import DatasetInfosDict -from datasets.load import (BuilderConfigsParameters, DatasetModule, +from datasets.load import (BuilderConfigsParameters, + DatasetModule, create_builder_configs_from_metadata_configs, - get_dataset_builder_class, import_main_class, + import_main_class, infer_module_for_data_files) from datasets.naming import camelcase_to_snakecase from datasets.packaged_modules import (_MODULE_TO_EXTENSIONS, _PACKAGED_DATASETS_MODULES) -from datasets.utils.file_utils import (cached_path, is_local_path, - relative_to_absolute_path) +from datasets.utils.file_utils import (cached_path, is_local_path) from datasets.utils.metadata import MetadataConfigs -from datasets.utils.track import tracked_str from fsspec import filesystem from fsspec.core import _un_chain from fsspec.utils import stringify_path @@ -42,14 +41,16 @@ from huggingface_hub import DatasetCard, DatasetCardData from packaging import version from modelscope import HubApi -from modelscope.msdatasets.utils._compat import ( - _HAS_SCRIPT_LOADING, _create_importable_file, _get_importable_file_path, - _load_importable_file, files_to_hash, get_imports, init_dynamic_modules, - resolve_trust_remote_code) +from modelscope.msdatasets.utils._compat import (_create_importable_file, + _get_importable_file_path, + _load_importable_file, + files_to_hash, + get_imports, + init_dynamic_modules, + resolve_trust_remote_code) from modelscope.utils.constant import (DEFAULT_DATASET_REVISION, REPO_TYPE_DATASET) from modelscope.utils.file_utils import is_relative_path -from modelscope.utils.import_utils import has_attr_in_class from modelscope.utils.logger import get_logger # ALL_ALLOWED_EXTENSIONS moved to datasets.packaged_modules in datasets 4.0 @@ -60,6 +61,73 @@ except ImportError: logger = get_logger() + +def _extract_split_names(split): + """Extract base split names from a split specification string. + + Handles simple names ("tool"), sliced splits ("train[:100]"), + and combined splits ("train+test"). + + Args: + split: A split specification string, or None. + + Returns: + A set of split name strings, or None if *split* is None or + cannot be parsed. + """ + if split is None: + return None + split_str = str(split) + parts = split_str.split('+') + names = set() + for part in parts: + # Remove slice notation like "[:100]" or "[50%:]" + name = re.split(r'\[', part.strip())[0] + if name: + names.add(name) + return names if names else None + + +def _filter_data_files_by_split(data_files, download_config): + """Filter data_files entries to only include the requested split(s). + + Args: + data_files: The raw data_files value from metadata_configs. + Expected format: list of dicts with 'split' and 'path' keys. + download_config: The DownloadConfig instance (may be None). + + Returns: + Filtered data_files if a split filter is active; otherwise + the original *data_files* unchanged. + """ + # 1. Safely retrieve the split value + split_str = None + if download_config is not None: + storage_opts = getattr(download_config, 'storage_options', None) + if isinstance(storage_opts, dict): + split_str = storage_opts.get('split') + + if split_str is None: + return data_files + + # 2. Parse split names + split_names = _extract_split_names(split_str) + if not split_names: + return data_files + + # 3. Only filter list[dict] format + if not isinstance(data_files, list): + return data_files + if not all(isinstance(item, dict) and 'split' in item for item in data_files): + return data_files + + # 4. Filter + filtered = [df for df in data_files if df.get('split') in split_names] + + # 5. Fallback: if filtered is empty, return original data + return filtered if filtered else data_files + + # --------------------------------------------------------------------------- # Shared HubApi instance (avoids creating a new requests.Session per call) # --------------------------------------------------------------------------- @@ -575,6 +643,8 @@ def get_module_without_script(self) -> DatasetModule: else: subset_data_files = next(iter( metadata_configs.values()))['data_files'] + subset_data_files = _filter_data_files_by_split( + subset_data_files, self.download_config) patterns = sanitize_patterns(subset_data_files) else: patterns = _get_data_patterns( diff --git a/modelscope/msdatasets/utils/hf_datasets_util.py b/modelscope/msdatasets/utils/hf_datasets_util.py index 1290cb26..8c89ecc9 100644 --- a/modelscope/msdatasets/utils/hf_datasets_util.py +++ b/modelscope/msdatasets/utils/hf_datasets_util.py @@ -12,6 +12,7 @@ Sub-modules: _module_factories – dataset module factory functions & data-file resolution """ import contextlib +import inspect import os import warnings from dataclasses import fields @@ -25,7 +26,7 @@ from datasets import (Dataset, DatasetBuilder, DatasetDict, DownloadConfig, DownloadManager, DownloadMode, Features, IterableDataset, IterableDatasetDict, Split, VerificationMode, Version, config, data_files, LargeList, - Sequence as SequenceHf) + Sequence as SequenceHf, SplitDict) try: from datasets import List as DatasetList @@ -407,10 +408,26 @@ def _get_paths_info( # =================================================================== -# HfFileSystem patch (_hf_fs_open) +# HfFileSystem patches (_hf_fs_open, _hf_fs_init) # =================================================================== _hf_fs_open_original = None +_hf_fs_init_original = None + + +def _hf_fs_init_with_cookie(self, *args, endpoint=None, token=None, **kwargs): + """HfFileSystem.__init__ wrapper that injects ModelScope cookie auth. + + ModelScope's /resolve/ endpoint authenticates via `m_session_id` cookie + rather than the `Authorization: Bearer` header used by HuggingFace Hub. + This wrapper ensures the cookie is included in all subsequent HTTP + requests made by the HfFileSystem instance. + """ + _hf_fs_init_original(self, *args, endpoint=endpoint, token=token, **kwargs) + if token and isinstance(token, str): + if not hasattr(self._api, 'headers') or self._api.headers is None: + self._api.headers = {} + self._api.headers['Cookie'] = f'm_session_id={token}' def _hf_fs_open(self, path, mode='rb', **kwargs): @@ -456,6 +473,154 @@ def _hf_fs_open(self, path, mode='rb', **kwargs): return _hf_fs_open_original(self, path, mode=mode, **kwargs) +class _DryRunDownloadManager: + """Minimal download-manager stub for split discovery without I/O. + + Returns placeholder paths for all download/extract calls, allowing + _split_generators() to execute and return SplitGenerator objects + (which carry split names) without triggering actual network or disk I/O. + """ + + _PLACEHOLDER = os.devnull + + def download(self, url_or_urls): + return self._map(url_or_urls) + + def download_and_extract(self, url_or_urls): + return self._map(url_or_urls) + + def extract(self, path_or_paths): + return self._map(path_or_paths) + + def download_custom(self, url_or_urls, custom_download): + return self._map(url_or_urls) + + def iter_archive(self, path): + return iter([]) + + def iter_files(self, paths): + return iter([]) + + @property + def manual_dir(self): + return None + + @property + def is_streaming(self): + return False + + def _map(self, input_): + if isinstance(input_, dict): + return {k: self._PLACEHOLDER for k in input_} + if isinstance(input_, (list, tuple, set)): + return type(input_)(self._PLACEHOLDER for _ in input_) + return self._PLACEHOLDER + + +def _discover_splits_from_builder(builder_instance): + """Discover available split names by dry-running _split_generators(). + + For script-based datasets that lack README split metadata, this calls + the builder's _split_generators() with a stub download manager to + extract split names without performing any actual downloads. + + Returns: + A set of split name strings, or an empty set if discovery fails. + """ + if not hasattr(builder_instance, '_split_generators'): + return set() + try: + generators = builder_instance._split_generators(_DryRunDownloadManager()) + return {str(sg.name) for sg in generators} + except Exception as e: + logger.debug(f'Failed to discover splits from builder: {e}') + return set() + + +def _validate_split_exists(builder_instance, split): + """Fail-fast check: raise ValueError before downloading if the + requested split does not exist in the dataset metadata. + + Args: + builder_instance: The DatasetBuilder instance with info/config. + split: The user-requested split specification (may be None). + + Raises: + ValueError: If any requested split name is not found among + the available splits declared in the dataset metadata. + """ + if split is None: + return + + from modelscope.msdatasets.utils._module_factories import _extract_split_names + split_names = _extract_split_names(split) + if not split_names: + return + + # Source 1: info.splits (original metadata from README) + available = set() + info = getattr(builder_instance, 'info', None) + if info is not None and info.splits: + available = set(info.splits.keys()) + + # Source 2: config.data_files keys + if not available: + config = getattr(builder_instance, 'config', None) + data_files = getattr(config, 'data_files', None) + if isinstance(data_files, dict): + available = set(data_files.keys()) + + # Source 3: dry-run _split_generators() for script-based datasets + if not available: + available = _discover_splits_from_builder(builder_instance) + + if not available: + logger.debug( + 'Cannot determine available splits from dataset metadata; ' + 'split validation skipped. Invalid splits will be caught downstream.' + ) + return + + missing = split_names - available + if missing: + raise ValueError( + f'Split {sorted(missing)} not found in dataset. ' + f'Available splits: {sorted(available)}' + ) + + +def _align_builder_splits_with_data_files(builder_instance, split): + """Align builder.info.splits with the actually requested split(s). + + When data_files have been filtered to a subset of splits (see + _filter_data_files_by_split in _module_factories.py), the builder's + info.splits metadata may still list all original splits from the + README. download_and_prepare() calls verify_splits() which would + then raise ExpectedMoreSplitsError. This helper prunes info.splits + to only contain the splits that will actually be generated. + """ + if split is None: + return + info = getattr(builder_instance, 'info', None) + if info is None or info.splits is None: + return + + from modelscope.msdatasets.utils._module_factories import _extract_split_names + split_names = _extract_split_names(split) + if not split_names: + return + + existing_keys = set(info.splits.keys()) + if split_names >= existing_keys: + return # All splits requested, no filtering needed + + filtered = {k: v for k, v in info.splits.items() if k in split_names} + if not filtered: + return # Safety: don't empty out splits + + info.splits = SplitDict(filtered, dataset_name=info.splits.dataset_name) + + # =================================================================== # DatasetsWrapperHF # =================================================================== @@ -544,34 +709,66 @@ class DatasetsWrapperHF: storage_options=storage_options, trust_remote_code=trust_remote_code, _require_default_config_name=name is None, + split=split, **config_kwargs, ) if dataset_info_only: ret_dict = {} + + # Case 1: Local .py script file if isinstance(path, str) and path.endswith('.py') and os.path.exists(path): from datasets import get_dataset_config_names subset_list = get_dataset_config_names(path) ret_dict = {_subset: [] for _subset in subset_list} return ret_dict - if builder_instance is None or not hasattr(builder_instance, - 'builder_configs'): - logger.error(f'No builder_configs found for {path} dataset.') + if builder_instance is None: + logger.error(f'No builder instance created for {path} dataset.') return ret_dict - _tmp_builder_configs = builder_instance.builder_configs - for tmp_config_name, tmp_builder_config in _tmp_builder_configs.items(): - tmp_config_name = str(tmp_config_name) - if hasattr(tmp_builder_config, 'data_files') and tmp_builder_config.data_files is not None: - ret_dict[tmp_config_name] = [str(item) for item in list(tmp_builder_config.data_files.keys())] + # Case 2: Try builder_configs with data_files (packaged datasets) + _tmp_builder_configs = getattr(builder_instance, 'builder_configs', None) + if _tmp_builder_configs: + if hasattr(_tmp_builder_configs, 'items'): + configs_iter = _tmp_builder_configs.items() else: - ret_dict[tmp_config_name] = [] + configs_iter = [(getattr(c, 'name', 'default'), c) for c in _tmp_builder_configs] + for tmp_config_name, tmp_builder_config in configs_iter: + tmp_config_name = str(tmp_config_name) + if hasattr(tmp_builder_config, 'data_files') and tmp_builder_config.data_files is not None: + ret_dict[tmp_config_name] = [str(item) for item in list(tmp_builder_config.data_files.keys())] + + # Case 3: Fallback for script datasets — use info.splits or dry-run discovery + if not ret_dict or all(not v for v in ret_dict.values()): + config_name = getattr(builder_instance, 'config_name', 'default') or 'default' + splits = [] + + # Try info.splits (from README metadata) + info = getattr(builder_instance, 'info', None) + if info is not None and info.splits: + splits = sorted(info.splits.keys()) + + # Fallback: dry-run _split_generators() + if not splits: + discovered = _discover_splits_from_builder(builder_instance) + if discovered: + splits = sorted(discovered) + + if splits: + ret_dict = {config_name: splits} + elif not ret_dict: + ret_dict = {config_name: []} + return ret_dict + _validate_split_exists(builder_instance, split) + if streaming: return builder_instance.as_streaming_dataset(split=split) + _align_builder_splits_with_data_files(builder_instance, split) + builder_instance.download_and_prepare( download_config=download_config, download_mode=download_mode, @@ -624,6 +821,7 @@ class DatasetsWrapperHF: storage_options: Optional[Dict] = None, trust_remote_code: Optional[bool] = None, _require_default_config_name=True, + split: Optional[Union[str, Split]] = None, **config_kwargs, ) -> DatasetBuilder: @@ -644,6 +842,12 @@ class DatasetsWrapperHF: download_config = download_config.copy( ) if download_config else DownloadConfig() download_config.storage_options.update(storage_options) + if split is not None: + download_config = download_config.copy( + ) if download_config else DownloadConfig() + if download_config.storage_options is None: + download_config.storage_options = {} + download_config.storage_options['split'] = split dataset_module = DatasetsWrapperHF.dataset_module_factory( path, @@ -683,6 +887,24 @@ class DatasetsWrapperHF: builder_cls = get_dataset_builder_class( dataset_module, dataset_name=dataset_name) + _config_cls = builder_cls.BUILDER_CONFIG_CLASS + if hasattr(_config_cls, '__dataclass_fields__'): + _valid_fields = set(_config_cls.__dataclass_fields__.keys()) + # Also preserve parameters accepted by the builder's + # __init__ (e.g. writer_batch_size, base_path, repo_id) + # so they are not inadvertently stripped. + try: + _init_params = set( + inspect.signature(builder_cls.__init__).parameters.keys() + ) + except (ValueError, TypeError): + _init_params = set() + _valid_fields = _valid_fields | _init_params + config_kwargs = { + k: v for k, v in config_kwargs.items() + if k in _valid_fields + } + builder_instance: DatasetBuilder = builder_cls( cache_dir=cache_dir, dataset_name=dataset_name, @@ -946,7 +1168,7 @@ def load_dataset_with_ctx(*args, **kwargs): non-streaming mode) or kept alive (for streaming mode, where lazy iteration needs the patches to remain active). """ - global _hf_fs_open_original + global _hf_fs_open_original, _hf_fs_init_original # Save originals hf_endpoint_origin = config.HF_ENDPOINT @@ -962,6 +1184,7 @@ def load_dataset_with_ctx(*args, **kwargs): HubDatasetModuleFactoryWithScript.get_module if _HAS_SCRIPT_LOADING else None) generate_from_dict_origin = features.generate_from_dict hf_fs_open_origin = HfFileSystem._open + hf_fs_init_origin = HfFileSystem.__init__ # Apply patches config.HF_ENDPOINT = get_endpoint() @@ -980,6 +1203,8 @@ def load_dataset_with_ctx(*args, **kwargs): features.generate_from_dict = generate_from_dict_ms _hf_fs_open_original = hf_fs_open_origin HfFileSystem._open = _hf_fs_open + _hf_fs_init_original = hf_fs_init_origin + HfFileSystem.__init__ = _hf_fs_init_with_cookie streaming = kwargs.get('streaming', False) @@ -990,10 +1215,12 @@ def load_dataset_with_ctx(*args, **kwargs): _repo_tree_cache.clear() HubApi._dataset_id_type_cache.clear() - HfFileSystem._open = hf_fs_open_origin - _hf_fs_open_original = None - if not streaming: + HfFileSystem._open = hf_fs_open_origin + _hf_fs_open_original = None + HfFileSystem.__init__ = hf_fs_init_origin + _hf_fs_init_original = None + config.HF_ENDPOINT = hf_endpoint_origin file_utils.get_from_cache = get_from_cache_origin features.generate_from_dict = generate_from_dict_origin diff --git a/modelscope/msdatasets/utils/hf_file_utils.py b/modelscope/msdatasets/utils/hf_file_utils.py index 2424dc50..8c30a359 100644 --- a/modelscope/msdatasets/utils/hf_file_utils.py +++ b/modelscope/msdatasets/utils/hf_file_utils.py @@ -53,11 +53,12 @@ def _request_with_retry_ms( url: str, max_retries: int = 2, base_wait_time: float = 0.5, - max_wait_time: float = 2, + max_wait_time: float = 3, timeout: float = 10.0, **params, ) -> requests.Response: - """Wrapper around requests to retry in case it fails with a ConnectTimeout, with exponential backoff. + """Wrapper around requests to retry in case it fails with a ConnectTimeout, + ReadTimeout or ConnectionError, with exponential backoff. Note that if the environment variable HF_DATASETS_OFFLINE is set to 1, then a OfflineModeIsEnabled error is raised. @@ -72,12 +73,35 @@ def _request_with_retry_ms( """ tries, success = 0, False response = None + range_header = (params.get('headers') or {}).get('Range', '') while not success: tries += 1 try: + logger.debug( + '[MS_DOWNLOAD] _request_with_retry_ms sending request: ' + 'method=%s, url=%s, timeout=%s, Range=%s', + method, url, timeout, range_header or 'N/A', + ) + t0 = time.perf_counter() response = requests.request(method=method.upper(), url=url, timeout=timeout, **params) + elapsed = time.perf_counter() - t0 + logger.debug( + '[MS_DOWNLOAD] _request_with_retry_ms response: ' + 'status=%s, content_length=%s, elapsed=%.3fs, url=%s', + response.status_code, + response.headers.get('Content-Length', 'N/A'), + elapsed, + url, + ) success = True - except (requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError) as err: + except (requests.exceptions.ReadTimeout, + requests.exceptions.ConnectTimeout, + requests.exceptions.ConnectionError) as err: + logger.error( + '[MS_DOWNLOAD] _request_with_retry_ms %s: ' + 'method=%s, url=%s, timeout=%s, error=%s', + type(err).__name__, method, url, timeout, err, + ) if tries > max_retries: raise err else: @@ -88,7 +112,7 @@ def _request_with_retry_ms( def http_head_ms( - url, proxies=None, headers=None, cookies=None, allow_redirects=True, timeout=10.0, max_retries=0 + url, proxies=None, headers=None, cookies=None, allow_redirects=True, timeout=10.0, max_retries=3 ) -> requests.Response: headers = copy.deepcopy(headers) or {} headers['user-agent'] = get_datasets_user_agent_ms(user_agent=headers.get('user-agent')) @@ -106,8 +130,12 @@ def http_head_ms( def http_get_ms( - url, temp_file, proxies=None, resume_size=0, headers=None, cookies=None, timeout=100.0, max_retries=0, desc=None + url, temp_file, proxies=None, resume_size=0, headers=None, cookies=None, timeout=300.0, max_retries=3, desc=None ) -> Optional[requests.Response]: + logger.debug( + '[MS_DOWNLOAD] http_get_ms entry: url=%s, timeout=%s, resume_size=%s', + url, timeout, resume_size, + ) headers = dict(headers) if headers is not None else {} headers['user-agent'] = get_datasets_user_agent_ms(user_agent=headers.get('user-agent')) if resume_size > 0: @@ -147,7 +175,7 @@ def get_from_cache_ms( user_agent=None, local_files_only=False, use_etag=True, - max_retries=0, + max_retries=3, token=None, use_auth_token='deprecated', ignore_url_params=False, @@ -320,6 +348,7 @@ def get_from_cache_ms( if scheme not in ('http', 'https'): fsspec_get(url, temp_file, storage_options=storage_options, desc=download_desc) else: + logger.info('[MS_DOWNLOAD] get_from_cache_ms downloading: url=%s', url) http_get_ms( url, temp_file=temp_file, diff --git a/modelscope/pipelines/audio/linear_aec_pipeline.py b/modelscope/pipelines/audio/linear_aec_pipeline.py index af0c2156..fb399476 100644 --- a/modelscope/pipelines/audio/linear_aec_pipeline.py +++ b/modelscope/pipelines/audio/linear_aec_pipeline.py @@ -70,7 +70,8 @@ class LinearAECPipeline(Pipeline): self.check_trust_remote_code( 'This pipeline requires `trust_remote_code=True` to load the module defined' ' in the `dey_mini.yaml`, setting this to True means you trust the code and files' - ' listed in this model repo.') + ' listed in this model repo.', + model_dir=model) self.use_cuda = torch.cuda.is_available() with open( diff --git a/modelscope/pipelines/builder.py b/modelscope/pipelines/builder.py index 5b2d1481..d379e3db 100644 --- a/modelscope/pipelines/builder.py +++ b/modelscope/pipelines/builder.py @@ -6,13 +6,15 @@ from typing import Any, Dict, List, Optional, Union from modelscope.hub.snapshot_download import snapshot_download from modelscope.metainfo import DEFAULT_MODEL_FOR_PIPELINE from modelscope.models.base import Model +from modelscope.utils.automodel_utils import check_model_from_owner_group from modelscope.utils.config import ConfigDict, check_config from modelscope.utils.constant import (DEFAULT_MODEL_REVISION, Invoke, Tasks, ThirdParty) from modelscope.utils.hub import read_config from modelscope.utils.import_utils import is_transformers_available from modelscope.utils.logger import get_logger -from modelscope.utils.plugins import (register_modelhub_repo, +from modelscope.utils.plugins import (filter_plugin_in_whitelist, + register_modelhub_repo, register_plugins_repo) from modelscope.utils.registry import Registry, build_from_cfg from modelscope.utils.task_utils import is_embedding_task @@ -79,6 +81,7 @@ def pipeline(task: str = None, device: str = None, model_revision: Optional[str] = DEFAULT_MODEL_REVISION, ignore_file_pattern: List[str] = None, + trust_remote_code: bool = False, **kwargs) -> Pipeline: """ Factory method to build an obj:`Pipeline`. @@ -95,6 +98,8 @@ def pipeline(task: str = None, device (str, optional): whether to use gpu or cpu is used to do inference. ignore_file_pattern(`str` or `List`, *optional*, default to `None`): Any file pattern to be ignored in downloading, like exact file names or file extensions. + trust_remote_code (bool, optional): Whether to allow execution of remote code or + plugins declared in the model configuration. Defaults to False. Return: pipeline (obj:`Pipeline`): pipeline object for certain task. @@ -113,6 +118,10 @@ def pipeline(task: str = None, if task is None and pipeline_name is None: raise ValueError('task or pipeline_name is required') + model_id = model[0] if isinstance(model, + list) and len(model) > 0 else model + _model_trusted = check_model_from_owner_group(model_id) + trust_remote_code = trust_remote_code or _model_trusted pipeline_props = None if pipeline_name is None: # get default pipeline for this task @@ -155,9 +164,24 @@ def pipeline(task: str = None, third_party=third_party, ignore_file_pattern=ignore_file_pattern) - register_plugins_repo(cfg.safe_get('plugins')) - register_modelhub_repo(model, - cfg.get('allow_remote', False)) + if cfg: + plugins = cfg.safe_get('plugins') + allow_remote = cfg.get('allow_remote', False) + if (filter_plugin_in_whitelist(plugins) + or allow_remote) and not trust_remote_code: + raise RuntimeError( + 'Detected plugins or allow_remote field in the model ' + 'configuration file, but trust_remote_code=True was not ' + 'explicitly set.\n' + 'To prevent potential execution of malicious code, loading ' + 'has been refused.\n' + 'If you trust this model repository, please pass ' + 'trust_remote_code=True to pipeline().') + register_plugins_repo(plugins) + model_dir = model if isinstance(model, + str) else model[0] + register_modelhub_repo( + model_dir, trust_remote_code and allow_remote) if pipeline_name: pipeline_props = {'type': pipeline_name} @@ -226,7 +250,13 @@ def pipeline(task: str = None, if preprocessor is not None: cfg.preprocessor = preprocessor - return build_pipeline(cfg, task_name=task) + if _model_trusted: + return build_pipeline(cfg, task_name=task) + else: + return build_pipeline( + cfg, + task_name=task, + default_args={'trust_remote_code': trust_remote_code}) def add_default_pipeline_info(task: str, diff --git a/modelscope/pipelines/multi_modal/disco_guided_diffusion_pipeline/disco_guided_diffusion.py b/modelscope/pipelines/multi_modal/disco_guided_diffusion_pipeline/disco_guided_diffusion.py index 670efae2..ebe4f0c2 100644 --- a/modelscope/pipelines/multi_modal/disco_guided_diffusion_pipeline/disco_guided_diffusion.py +++ b/modelscope/pipelines/multi_modal/disco_guided_diffusion_pipeline/disco_guided_diffusion.py @@ -193,7 +193,8 @@ class DiscoDiffusionPipeline(DiffusersPipeline): self.check_trust_remote_code( 'This pipeline requires `trust_remote_code=True` to load the module defined' ' in `model_index.json`, setting this to True means you trust the code and files' - ' listed in this model repo.') + ' listed in this model repo.', + model_dir=model) model_path = model @@ -209,12 +210,6 @@ class DiscoDiffusionPipeline(DiffusersPipeline): if model_config['use_fp16']: self.unet.convert_to_fp16() - self.trust_remote_code = kwargs.get('trust_remote_code', False) - self.check_trust_remote_code( - 'This pipeline requires import modules listed in `model_index.json`, ' - 'please add `trust_remote_code=True` if you trust this model repo.' - ) - with open( os.path.join(model_path, 'model_index.json'), 'r', diff --git a/modelscope/preprocessors/base.py b/modelscope/preprocessors/base.py index d3138a3e..f87cebe7 100644 --- a/modelscope/preprocessors/base.py +++ b/modelscope/preprocessors/base.py @@ -208,6 +208,7 @@ class Preprocessor(ABC): revision: Optional[str] = DEFAULT_MODEL_REVISION, cfg_dict: Config = None, preprocessor_mode=ModeKeys.INFERENCE, + trust_remote_code=False, **kwargs): """Instantiate a preprocessor from local directory or remote model repo. Note that when loading from remote, the model revision can be specified. diff --git a/modelscope/trainers/builder.py b/modelscope/trainers/builder.py index 8d1847f6..fdef22f4 100644 --- a/modelscope/trainers/builder.py +++ b/modelscope/trainers/builder.py @@ -2,10 +2,11 @@ from modelscope.metainfo import Trainers from modelscope.pipelines.builder import normalize_model_input from modelscope.pipelines.util import is_official_hub_path -from modelscope.utils.config import check_config +from modelscope.utils.automodel_utils import check_model_from_owner_group from modelscope.utils.constant import DEFAULT_MODEL_REVISION from modelscope.utils.hub import read_config -from modelscope.utils.plugins import (register_modelhub_repo, +from modelscope.utils.plugins import (filter_plugin_in_whitelist, + register_modelhub_repo, register_plugins_repo) from modelscope.utils.registry import Registry, build_from_cfg @@ -19,10 +20,18 @@ def build_trainer(name: str = Trainers.default, default_args: dict = None): name (str, optional): Trainer name, if None, default trainer will be used. default_args (dict, optional): Default initialization arguments. + If ``trust_remote_code`` key is set to True in default_args, + remote code and plugins declared in the model configuration + will be allowed to execute. """ cfg = dict(type=name) + default_args = default_args or {} model = default_args.get('model', None) model_revision = default_args.get('model_revision', DEFAULT_MODEL_REVISION) + model_id = model[0] if isinstance(model, + list) and len(model) > 0 else model + trust_remote_code = default_args.get( + 'trust_remote_code', False) or check_model_from_owner_group(model_id) if isinstance(model, str) \ or (isinstance(model, list) and isinstance(model[0], str)): @@ -33,7 +42,23 @@ def build_trainer(name: str = Trainers.default, default_args: dict = None): model, str) else read_config( model[0], revision=model_revision) model_dir = normalize_model_input(model, model_revision) - register_plugins_repo(configuration.safe_get('plugins')) - register_modelhub_repo(model_dir, - configuration.get('allow_remote', False)) + if configuration: + plugins = configuration.safe_get('plugins') + allow_remote = configuration.get('allow_remote', False) + if (filter_plugin_in_whitelist(plugins) + or allow_remote) and not trust_remote_code: + raise RuntimeError( + 'Detected plugins or allow_remote field in the model ' + 'configuration file, but trust_remote_code=True was ' + 'not explicitly set.\n' + 'To prevent potential execution of malicious code, ' + 'loading has been refused.\n' + 'If you trust this model repository, please pass ' + 'trust_remote_code=True in default_args to ' + 'build_trainer().') + register_plugins_repo(plugins) + model_dir_str = model_dir if isinstance(model_dir, + str) else model_dir[0] + register_modelhub_repo(model_dir_str, trust_remote_code + and allow_remote) return build_from_cfg(cfg, TRAINERS, default_args=default_args) diff --git a/modelscope/trainers/trainer.py b/modelscope/trainers/trainer.py index 25f948bc..ca11cad3 100644 --- a/modelscope/trainers/trainer.py +++ b/modelscope/trainers/trainer.py @@ -135,6 +135,7 @@ class EpochBasedTrainer(BaseTrainer): self._inner_iter = 0 self._stop_training = False self._compile = kwargs.get('compile', False) + self.trust_remote_code = kwargs.get('trust_remote_code', False) self.train_dataloader = None self.eval_dataloader = None @@ -814,7 +815,10 @@ class EpochBasedTrainer(BaseTrainer): override this method in a subclass. """ - model = Model.from_pretrained(self.model_dir, cfg_dict=self.cfg) + model = Model.from_pretrained( + self.model_dir, + cfg_dict=self.cfg, + trust_remote_code=self.trust_remote_code) if not isinstance(model, nn.Module) and hasattr(model, 'model'): return model.model elif isinstance(model, nn.Module): diff --git a/modelscope/utils/automodel_utils.py b/modelscope/utils/automodel_utils.py index bf68da74..ba087b22 100644 --- a/modelscope/utils/automodel_utils.py +++ b/modelscope/utils/automodel_utils.py @@ -137,7 +137,7 @@ def check_model_from_owner_group(model_dir: str, Returns: bool: Whether the group can be trusted """ - if not model_dir: + if not model_dir or not isinstance(model_dir, str): return False if owner_group is None: owner_group = ['iic', 'damo'] diff --git a/modelscope/utils/hf_util/patcher.py b/modelscope/utils/hf_util/patcher.py index 8aadb596..59fefee0 100644 --- a/modelscope/utils/hf_util/patcher.py +++ b/modelscope/utils/hf_util/patcher.py @@ -524,6 +524,7 @@ def _patch_kernels(): """ try: from kernels import utils as kernels_utils + from kernels.utils import _get_hf_api except ImportError: return if hasattr(kernels_utils, '_get_hf_api_origin'): @@ -535,6 +536,7 @@ def _patch_kernels(): def _unpatch_kernels(): try: from kernels import utils as kernels_utils + from kernels.utils import _get_hf_api except ImportError: return origin = getattr(kernels_utils, '_get_hf_api_origin', None) diff --git a/modelscope/utils/plugins.py b/modelscope/utils/plugins.py index c4957202..fa4cb877 100644 --- a/modelscope/utils/plugins.py +++ b/modelscope/utils/plugins.py @@ -8,6 +8,7 @@ import importlib import importlib.metadata import os import pkgutil +import re import shutil import subprocess import sys @@ -46,6 +47,18 @@ OFFICIAL_PLUGINS = [ LOCAL_PLUGINS_FILENAME = '.modelscope_plugins' GLOBAL_PLUGINS_FILENAME = os.path.join(Path.home(), '.modelscope', 'plugins') DEFAULT_PLUGINS = [] +PLUGIN_WHITE_LIST = ['pai-easycv'] + + +def filter_plugin_in_whitelist(plugins): + if not plugins: + return plugins + if isinstance(plugins, str): + plugins = [plugins] + return [ + plugin for plugin in plugins if re.split(r'[><=!~]', plugin.strip()) + [0].strip() not in PLUGIN_WHITE_LIST + ] @contextmanager diff --git a/modelscope/utils/registry.py b/modelscope/utils/registry.py index ae81cfc4..a323d2a8 100644 --- a/modelscope/utils/registry.py +++ b/modelscope/utils/registry.py @@ -205,6 +205,8 @@ def build_from_cfg(cfg, raise TypeError( f'type must be a str or valid type, but got {type(obj_type)}') try: + if not obj_cls.__module__.startswith('modelscope'): + args.pop('trust_remote_code', None) if hasattr(obj_cls, '_instantiate'): return obj_cls._instantiate(**args) else: diff --git a/modelscope/version.py b/modelscope/version.py index 6696a5f1..1105d343 100644 --- a/modelscope/version.py +++ b/modelscope/version.py @@ -1,5 +1,5 @@ # Make sure to modify __release_datetime__ to release time when making official release. -__version__ = '1.36.3' +__version__ = '1.37.1' # default release datetime for branches under active development is set # to be a time far-far-away-into-the-future -__release_datetime__ = '2026-04-29 02:00:00' +__release_datetime__ = '2026-05-20 23:59:59'