From c18f11ccbdd16835e9925123364cedee25dc2eb3 Mon Sep 17 00:00:00 2001 From: addsubmuldiv Date: Wed, 29 Apr 2026 16:06:57 +0800 Subject: [PATCH 01/23] update ascend dockerfile (#1687) --- docker/Dockerfile.ascend | 62 +++++++++++++++++++++++++++++++++------- docker/build_image.py | 47 ++++++++++++++++++++++++++++-- 2 files changed, 95 insertions(+), 14 deletions(-) diff --git a/docker/Dockerfile.ascend b/docker/Dockerfile.ascend index b18eff2f..c9f4e80e 100644 --- a/docker/Dockerfile.ascend +++ b/docker/Dockerfile.ascend @@ -1,14 +1,13 @@ FROM {base_image} -# Use bash so that `source` and other bash builtins work in all following RUN steps. +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_DEFAULT_TIMEOUT=300 \ + PIP_RETRIES=10 \ + SOC_VERSION={soc_version} + SHELL ["/bin/bash", "-c"] -ENV PIP_DISABLE_PIP_VERSION_CHECK=1 -ENV PIP_DEFAULT_TIMEOUT=300 -ENV PIP_RETRIES=10 -ENV TRANSFORMERS_VERBOSITY=error -ENV TRANSFORMERS_NO_ADVISORY_WARNINGS=1 - +# ---------- System dependencies ---------- RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ find /etc/apt/apt.conf.d -maxdepth 1 -type f | xargs -r grep -l "APT::Update::Post-Invoke\|docker-clean" | xargs -r rm -f && \ apt-get update -y && \ @@ -19,23 +18,62 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ rm -rf /var/lib/apt/lists/* RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \ - pip config set install.trusted-host mirrors.aliyun.com + pip config set install.trusted-host mirrors.aliyun.com && \ + ARCH=$(uname -m) && \ + if [ "$ARCH" = "x86_64" ]; then \ + pip config set global.extra-index-url "https://download.pytorch.org/whl/cpu/"; \ + fi {extra_content} +# ---------- Install vllm + vllm-ascend ---------- +RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ + if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \ + git clone --depth 1 --branch v0.14.0 https://github.com/vllm-project/vllm && \ + git clone --depth 1 --branch v0.14.0rc1 https://github.com/vllm-project/vllm-ascend.git -# Reuse the vllm-ascend base image and only add the extra repos we need. -# --depth 1 keeps the image smaller; branch/tag names work with shallow clone. +RUN ARCH=$(uname -m) && \ + source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ + source /usr/local/Ascend/nnal/atb/set_env.sh && \ + # Install torch & torch_npu & torchvision + pip install torch==2.9.0 torch_npu==2.9.0 torchvision==0.24.0 && \ + # Install vllm + cd vllm && VLLM_TARGET_DEVICE=empty pip install -v -e . && cd .. && \ + # Install vllm-ascend + cd vllm-ascend && pip install -v -e . && cd .. + +# ---------- Clone training-side repositories ---------- RUN git clone --depth 1 --branch v0.15.3 https://github.com/NVIDIA/Megatron-LM.git /Megatron-LM && \ git clone --depth 1 --branch core_r0.15.3 https://gitcode.com/Ascend/MindSpeed.git /MindSpeed && \ GIT_LFS_SKIP_SMUDGE=1 git clone --depth 1 -b {swift_branch} --single-branch https://github.com/modelscope/ms-swift.git /ms-swift && \ git clone --depth 1 https://github.com/modelscope/mcore-bridge.git /mcore-bridge +# ---------- Install training-side repositories ---------- RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \ cd /MindSpeed && pip install --no-cache-dir -e . && \ cd /mcore-bridge && pip install --no-cache-dir -e . && \ - pip cache purge + cd /ms-swift && pip install --no-cache-dir -e . +# ---------- Pin torch to the correct version + torch_npu ---------- +# x86: must force-install the CPU build from pytorch.org/whl/cpu +# aarch64: PyPI only provides the CPU build, so install it directly from the Aliyun mirror +RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ + if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \ + ARCH=$(uname -m) && \ + if [ "$ARCH" = "x86_64" ]; then \ + pip install --no-cache-dir --force-reinstall --no-deps \ + --index-url https://download.pytorch.org/whl/cpu \ + torch==2.9.0 torchvision==0.24.0 torchaudio==2.9.0; \ + else \ + pip install --no-cache-dir --force-reinstall --no-deps \ + torch==2.9.0 torchvision==0.24.0 torchaudio==2.9.0; \ + fi && \ + pip install --no-cache-dir --force-reinstall --no-deps \ + torch_npu==2.9.0 && \ + rm -rf /root/.cache/pip + +# ---------- Remove CUDA-only dependencies pulled in by vllm (they cause missing libtorch_cuda.so errors on NPU) ---------- +RUN pip uninstall -y flashinfer tvm-ffi torch-c-dlpack-ext 2>/dev/null || true ARG INSTALL_MS_DEPS={install_ms_deps} ENV MEGATRON_LM_PATH=/Megatron-LM @@ -68,6 +106,8 @@ fi ARG CUR_TIME={cur_time} RUN echo $CUR_TIME +RUN pip install --no-cache-dir --no-build-isolation OpenCC + RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \ pip install --no-cache-dir -U funasr scikit-learn && \ diff --git a/docker/build_image.py b/docker/build_image.py index 37d35787..a80879b0 100644 --- a/docker/build_image.py +++ b/docker/build_image.py @@ -1,5 +1,6 @@ import argparse import os +import platform import subprocess from datetime import datetime from typing import Any @@ -461,10 +462,46 @@ RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy class AscendImageBuilder(StableGPUImageBuilder): + @staticmethod + def _normalize_arch(arch: str = None) -> str: + arch = arch or platform.machine() + arch = arch.lower() + arch_mapping = { + 'x86': 'x86', + 'x86_64': 'x86', + 'amd64': 'x86', + 'arm': 'arm', + 'aarch64': 'arm', + 'arm64': 'arm', + } + if arch not in arch_mapping: + raise ValueError(f'Unsupported architecture: {arch}. ' + 'Please pass --arch x86 or --arch arm.') + return arch_mapping[arch] + + @staticmethod + def _get_atlas_hardware(soc_version: str) -> str: + soc_version = soc_version.lower() + atlas_mapping = { + 'ascend910b1': 'A2', + 'ascend910_9391': 'A3', + 'ascend310p1': '300I', + } + if soc_version.startswith('ascend950'): + return 'A5' + if soc_version not in atlas_mapping: + raise ValueError( + f'Unsupported soc_version: {soc_version}. ' + 'Supported values are ascend910b1, ascend910_9391, ' + 'ascend310p1, and values starting with ascend950.') + return atlas_mapping[soc_version] + def init_args(self, args) -> Any: if not args.base_image: # Reuse the prebuilt vllm-ascend image to avoid rebuilding its stack. - args.base_image = 'quay.io/ascend/vllm-ascend:v0.14.0rc1-a3' + args.base_image = 'quay.io/ascend/cann:8.5.1-a3-ubuntu22.04-py3.11' + args.arch = self._normalize_arch(args.arch) + args.atlas_hardware = self._get_atlas_hardware(args.soc_version) return super().init_args(args) def generate_dockerfile(self) -> str: @@ -474,6 +511,7 @@ RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy with open('docker/Dockerfile.ascend', 'r') as f: content = f.read() content = content.replace('{base_image}', self.args.base_image) + content = content.replace('{soc_version}', self.args.soc_version) content = content.replace('{extra_content}', extra_content) content = content.replace('{cur_time}', formatted_time) content = content.replace('{install_ms_deps}', 'False') @@ -484,8 +522,9 @@ RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy def image(self) -> str: return ( - f'{docker_registry}:{self.args.base_image.split(":")[-1]}-torch2.7.1' - f'-{self.args.modelscope_version}-ascend-test') + f'{docker_registry}:{self.args.swift_branch}-' + f'{self.args.atlas_hardware}-{self.args.python_tag}-{self.args.arch}' + ) def push(self): return 0 @@ -510,6 +549,8 @@ parser.add_argument('--optimum_version', type=str, default=None) parser.add_argument('--modelscope_branch', type=str, default='master') parser.add_argument('--modelscope_version', type=str, default='9.99.0') parser.add_argument('--swift_branch', type=str, default='main') +parser.add_argument('--soc_version', type=str, default='ascend910_9391') +parser.add_argument('--arch', type=str, choices=['x86', 'arm'], default=None) parser.add_argument('--dry_run', type=int, default=0) args = parser.parse_args() From 13064d9486d7f4be46294860767b1e93e149104f Mon Sep 17 00:00:00 2001 From: "Xingjun.Wang" Date: Tue, 5 May 2026 23:49:23 +0800 Subject: [PATCH 02/23] [Fix] Fix msdatasets split issue (#1704) --- modelscope/msdatasets/ms_dataset.py | 2 +- .../msdatasets/utils/_module_factories.py | 96 ++++++++++++++++--- .../msdatasets/utils/hf_datasets_util.py | 88 ++++++++++++++++- modelscope/msdatasets/utils/hf_file_utils.py | 15 +-- 4 files changed, 180 insertions(+), 21 deletions(-) diff --git a/modelscope/msdatasets/ms_dataset.py b/modelscope/msdatasets/ms_dataset.py index a18532e0..55ca949f 100644 --- a/modelscope/msdatasets/ms_dataset.py +++ b/modelscope/msdatasets/ms_dataset.py @@ -246,7 +246,7 @@ class MsDataset: 'you can trust the external codes.') # Raise csv field size limit to avoid errors with large cells - if config_kwargs.get('engine') == 'python': + if config_kwargs.pop('engine', None) == 'python': import csv as csv_module import sys try: diff --git a/modelscope/msdatasets/utils/_module_factories.py b/modelscope/msdatasets/utils/_module_factories.py index 861048ff..0dc12f27 100644 --- a/modelscope/msdatasets/utils/_module_factories.py +++ b/modelscope/msdatasets/utils/_module_factories.py @@ -9,12 +9,12 @@ by :func:`~hf_datasets_util.load_dataset_with_ctx`. import importlib import inspect import os +import re from functools import partial from pathlib import Path -from typing import Dict, List, Optional, Sequence, Tuple, Union +from typing import Dict, List, Optional, Tuple -from datasets import (BuilderConfig, DownloadConfig, DownloadMode, Features, - Version, config, data_files) +from datasets import (BuilderConfig, DownloadConfig, config) from datasets.data_files import ( FILES_TO_IGNORE, DataFilesDict, EmptyDatasetError, _get_data_files_patterns, _is_inside_unrequested_special_dir, @@ -24,17 +24,16 @@ from datasets.download.streaming_download_manager import ( _prepare_path_and_storage_options, xbasename, xjoin) from datasets.exceptions import DataFilesNotFoundError from datasets.info import DatasetInfosDict -from datasets.load import (BuilderConfigsParameters, DatasetModule, +from datasets.load import (BuilderConfigsParameters, + DatasetModule, create_builder_configs_from_metadata_configs, - get_dataset_builder_class, import_main_class, + import_main_class, infer_module_for_data_files) from datasets.naming import camelcase_to_snakecase from datasets.packaged_modules import (_MODULE_TO_EXTENSIONS, _PACKAGED_DATASETS_MODULES) -from datasets.utils.file_utils import (cached_path, is_local_path, - relative_to_absolute_path) +from datasets.utils.file_utils import (cached_path, is_local_path) from datasets.utils.metadata import MetadataConfigs -from datasets.utils.track import tracked_str from fsspec import filesystem from fsspec.core import _un_chain from fsspec.utils import stringify_path @@ -42,14 +41,16 @@ from huggingface_hub import DatasetCard, DatasetCardData from packaging import version from modelscope import HubApi -from modelscope.msdatasets.utils._compat import ( - _HAS_SCRIPT_LOADING, _create_importable_file, _get_importable_file_path, - _load_importable_file, files_to_hash, get_imports, init_dynamic_modules, - resolve_trust_remote_code) +from modelscope.msdatasets.utils._compat import (_create_importable_file, + _get_importable_file_path, + _load_importable_file, + files_to_hash, + get_imports, + init_dynamic_modules, + resolve_trust_remote_code) from modelscope.utils.constant import (DEFAULT_DATASET_REVISION, REPO_TYPE_DATASET) from modelscope.utils.file_utils import is_relative_path -from modelscope.utils.import_utils import has_attr_in_class from modelscope.utils.logger import get_logger # ALL_ALLOWED_EXTENSIONS moved to datasets.packaged_modules in datasets 4.0 @@ -60,6 +61,73 @@ except ImportError: logger = get_logger() + +def _extract_split_names(split): + """Extract base split names from a split specification string. + + Handles simple names ("tool"), sliced splits ("train[:100]"), + and combined splits ("train+test"). + + Args: + split: A split specification string, or None. + + Returns: + A set of split name strings, or None if *split* is None or + cannot be parsed. + """ + if split is None: + return None + split_str = str(split) + parts = split_str.split('+') + names = set() + for part in parts: + # Remove slice notation like "[:100]" or "[50%:]" + name = re.split(r'\[', part.strip())[0] + if name: + names.add(name) + return names if names else None + + +def _filter_data_files_by_split(data_files, download_config): + """Filter data_files entries to only include the requested split(s). + + Args: + data_files: The raw data_files value from metadata_configs. + Expected format: list of dicts with 'split' and 'path' keys. + download_config: The DownloadConfig instance (may be None). + + Returns: + Filtered data_files if a split filter is active; otherwise + the original *data_files* unchanged. + """ + # 1. Safely retrieve the split value + split_str = None + if download_config is not None: + storage_opts = getattr(download_config, 'storage_options', None) + if isinstance(storage_opts, dict): + split_str = storage_opts.get('split') + + if split_str is None: + return data_files + + # 2. Parse split names + split_names = _extract_split_names(split_str) + if not split_names: + return data_files + + # 3. Only filter list[dict] format + if not isinstance(data_files, list): + return data_files + if not all(isinstance(item, dict) and 'split' in item for item in data_files): + return data_files + + # 4. Filter + filtered = [df for df in data_files if df.get('split') in split_names] + + # 5. Fallback: if filtered is empty, return original data + return filtered if filtered else data_files + + # --------------------------------------------------------------------------- # Shared HubApi instance (avoids creating a new requests.Session per call) # --------------------------------------------------------------------------- @@ -575,6 +643,8 @@ def get_module_without_script(self) -> DatasetModule: else: subset_data_files = next(iter( metadata_configs.values()))['data_files'] + subset_data_files = _filter_data_files_by_split( + subset_data_files, self.download_config) patterns = sanitize_patterns(subset_data_files) else: patterns = _get_data_patterns( diff --git a/modelscope/msdatasets/utils/hf_datasets_util.py b/modelscope/msdatasets/utils/hf_datasets_util.py index 1290cb26..f422e064 100644 --- a/modelscope/msdatasets/utils/hf_datasets_util.py +++ b/modelscope/msdatasets/utils/hf_datasets_util.py @@ -25,7 +25,7 @@ from datasets import (Dataset, DatasetBuilder, DatasetDict, DownloadConfig, DownloadManager, DownloadMode, Features, IterableDataset, IterableDatasetDict, Split, VerificationMode, Version, config, data_files, LargeList, - Sequence as SequenceHf) + Sequence as SequenceHf, SplitDict) try: from datasets import List as DatasetList @@ -456,6 +456,81 @@ def _hf_fs_open(self, path, mode='rb', **kwargs): return _hf_fs_open_original(self, path, mode=mode, **kwargs) +def _validate_split_exists(builder_instance, split): + """Fail-fast check: raise ValueError before downloading if the + requested split does not exist in the dataset metadata. + + Args: + builder_instance: The DatasetBuilder instance with info/config. + split: The user-requested split specification (may be None). + + Raises: + ValueError: If any requested split name is not found among + the available splits declared in the dataset metadata. + """ + if split is None: + return + + from modelscope.msdatasets.utils._module_factories import _extract_split_names + split_names = _extract_split_names(split) + if not split_names: + return + + # Prefer info.splits (original metadata); fall back to data_files keys + available = set() + info = getattr(builder_instance, 'info', None) + if info is not None and info.splits: + available = set(info.splits.keys()) + + if not available: + config = getattr(builder_instance, 'config', None) + data_files = getattr(config, 'data_files', None) + if isinstance(data_files, dict): + available = set(data_files.keys()) + + if not available: + return # Cannot determine available splits; let downstream handle + + missing = split_names - available + if missing: + raise ValueError( + f'Split {sorted(missing)} not found in dataset. ' + f'Available splits: {sorted(available)}' + ) + + +def _align_builder_splits_with_data_files(builder_instance, split): + """Align builder.info.splits with the actually requested split(s). + + When data_files have been filtered to a subset of splits (see + _filter_data_files_by_split in _module_factories.py), the builder's + info.splits metadata may still list all original splits from the + README. download_and_prepare() calls verify_splits() which would + then raise ExpectedMoreSplitsError. This helper prunes info.splits + to only contain the splits that will actually be generated. + """ + if split is None: + return + info = getattr(builder_instance, 'info', None) + if info is None or info.splits is None: + return + + from modelscope.msdatasets.utils._module_factories import _extract_split_names + split_names = _extract_split_names(split) + if not split_names: + return + + existing_keys = set(info.splits.keys()) + if split_names >= existing_keys: + return # All splits requested, no filtering needed + + filtered = {k: v for k, v in info.splits.items() if k in split_names} + if not filtered: + return # Safety: don't empty out splits + + info.splits = SplitDict(filtered, dataset_name=info.splits.dataset_name) + + # =================================================================== # DatasetsWrapperHF # =================================================================== @@ -544,6 +619,7 @@ class DatasetsWrapperHF: storage_options=storage_options, trust_remote_code=trust_remote_code, _require_default_config_name=name is None, + split=split, **config_kwargs, ) @@ -572,6 +648,9 @@ class DatasetsWrapperHF: if streaming: return builder_instance.as_streaming_dataset(split=split) + _validate_split_exists(builder_instance, split) + _align_builder_splits_with_data_files(builder_instance, split) + builder_instance.download_and_prepare( download_config=download_config, download_mode=download_mode, @@ -624,6 +703,7 @@ class DatasetsWrapperHF: storage_options: Optional[Dict] = None, trust_remote_code: Optional[bool] = None, _require_default_config_name=True, + split: Optional[Union[str, Split]] = None, **config_kwargs, ) -> DatasetBuilder: @@ -644,6 +724,12 @@ class DatasetsWrapperHF: download_config = download_config.copy( ) if download_config else DownloadConfig() download_config.storage_options.update(storage_options) + if split is not None: + download_config = download_config.copy( + ) if download_config else DownloadConfig() + if download_config.storage_options is None: + download_config.storage_options = {} + download_config.storage_options['split'] = split dataset_module = DatasetsWrapperHF.dataset_module_factory( path, diff --git a/modelscope/msdatasets/utils/hf_file_utils.py b/modelscope/msdatasets/utils/hf_file_utils.py index 2424dc50..b8a327f6 100644 --- a/modelscope/msdatasets/utils/hf_file_utils.py +++ b/modelscope/msdatasets/utils/hf_file_utils.py @@ -53,11 +53,12 @@ def _request_with_retry_ms( url: str, max_retries: int = 2, base_wait_time: float = 0.5, - max_wait_time: float = 2, + max_wait_time: float = 8, timeout: float = 10.0, **params, ) -> requests.Response: - """Wrapper around requests to retry in case it fails with a ConnectTimeout, with exponential backoff. + """Wrapper around requests to retry in case it fails with a ConnectTimeout, + ReadTimeout or ConnectionError, with exponential backoff. Note that if the environment variable HF_DATASETS_OFFLINE is set to 1, then a OfflineModeIsEnabled error is raised. @@ -77,7 +78,9 @@ def _request_with_retry_ms( try: response = requests.request(method=method.upper(), url=url, timeout=timeout, **params) success = True - except (requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError) as err: + except (requests.exceptions.ConnectTimeout, + requests.exceptions.ConnectionError, + requests.exceptions.ReadTimeout) as err: if tries > max_retries: raise err else: @@ -88,7 +91,7 @@ def _request_with_retry_ms( def http_head_ms( - url, proxies=None, headers=None, cookies=None, allow_redirects=True, timeout=10.0, max_retries=0 + url, proxies=None, headers=None, cookies=None, allow_redirects=True, timeout=10.0, max_retries=3 ) -> requests.Response: headers = copy.deepcopy(headers) or {} headers['user-agent'] = get_datasets_user_agent_ms(user_agent=headers.get('user-agent')) @@ -106,7 +109,7 @@ def http_head_ms( def http_get_ms( - url, temp_file, proxies=None, resume_size=0, headers=None, cookies=None, timeout=100.0, max_retries=0, desc=None + url, temp_file, proxies=None, resume_size=0, headers=None, cookies=None, timeout=300.0, max_retries=3, desc=None ) -> Optional[requests.Response]: headers = dict(headers) if headers is not None else {} headers['user-agent'] = get_datasets_user_agent_ms(user_agent=headers.get('user-agent')) @@ -147,7 +150,7 @@ def get_from_cache_ms( user_agent=None, local_files_only=False, use_etag=True, - max_retries=0, + max_retries=3, token=None, use_auth_token='deprecated', ignore_url_params=False, From 68ab75af24ed3fa1184da98c5afe9c4e5780acdb Mon Sep 17 00:00:00 2001 From: tastelikefeet <58414341+tastelikefeet@users.noreply.github.com> Date: Wed, 6 May 2026 20:54:46 +0800 Subject: [PATCH 03/23] fix (#1708) --- modelscope/utils/hf_util/patcher.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modelscope/utils/hf_util/patcher.py b/modelscope/utils/hf_util/patcher.py index 8aadb596..59fefee0 100644 --- a/modelscope/utils/hf_util/patcher.py +++ b/modelscope/utils/hf_util/patcher.py @@ -524,6 +524,7 @@ def _patch_kernels(): """ try: from kernels import utils as kernels_utils + from kernels.utils import _get_hf_api except ImportError: return if hasattr(kernels_utils, '_get_hf_api_origin'): @@ -535,6 +536,7 @@ def _patch_kernels(): def _unpatch_kernels(): try: from kernels import utils as kernels_utils + from kernels.utils import _get_hf_api except ImportError: return origin = getattr(kernels_utils, '_get_hf_api_origin', None) From 2f5f52fc3cb6024c57049c143a05b88e7d65b875 Mon Sep 17 00:00:00 2001 From: dwd Date: Wed, 6 May 2026 20:57:17 +0800 Subject: [PATCH 04/23] feat(docker/Metax): add metax dockerfile and its requirements for ms-swift 4.1.x (#1689) --- docker/Metax/4.1/Dockerfile.metax | 164 ++++++++++++++++++ docker/Metax/4.1/Dockerfile.with_metax_image | 73 ++++++++ docker/Metax/4.1/build.sh | 13 ++ docker/Metax/4.1/build_from_metax_image.sh | 10 ++ docker/Metax/4.1/override.txt | 5 + docker/Metax/4.1/requirements_extra.txt | 14 ++ .../Metax/4.1/swift_building_instructions.md | 52 ++++++ 7 files changed, 331 insertions(+) create mode 100644 docker/Metax/4.1/Dockerfile.metax create mode 100644 docker/Metax/4.1/Dockerfile.with_metax_image create mode 100644 docker/Metax/4.1/build.sh create mode 100644 docker/Metax/4.1/build_from_metax_image.sh create mode 100644 docker/Metax/4.1/override.txt create mode 100644 docker/Metax/4.1/requirements_extra.txt create mode 100644 docker/Metax/4.1/swift_building_instructions.md diff --git a/docker/Metax/4.1/Dockerfile.metax b/docker/Metax/4.1/Dockerfile.metax new file mode 100644 index 00000000..07ec8205 --- /dev/null +++ b/docker/Metax/4.1/Dockerfile.metax @@ -0,0 +1,164 @@ +ARG BUILD_BASE_IMAGE=registry.access.redhat.com/ubi9/ubi:9.6 +ARG PYTHON_VERSION=3.12 +ARG UV_EXTRA_INDEX_URL=https://repos.metax-tech.com/r/maca-pypi/simple +ARG UV_TRUSTED_HOST=repos.metax-tech.com + +# may need passing a particular vllm version during build +ARG VLLM_VERSION +ARG MACA_VERSION +ARG CU_BRIDGE_VERSION=${MACA_VERSION} + +#################### BASE BUILD IMAGE #################### +FROM ${BUILD_BASE_IMAGE} AS base +ARG UV_TRUSTED_HOST + +# maca environment variables +ENV MACA_PATH=/opt/maca +ENV MACA_CLANG_PATH=/opt/maca/mxgpu_llvm/bin +ENV CUCC_PATH="${MACA_PATH}/tools/cu-bridge" +ENV CUDA_PATH=/root/cu-bridge/CUDA_DIR +ENV CUCC_CMAKE_ENTRY=2 +ENV PATH="/opt/venv/bin:/root/.local/bin:$PATH" +ENV PATH=/opt/mxdriver/bin:${MACA_PATH}/bin:${MACA_PATH}/mxgpu_llvm/bin:${MACA_PATH}/tools/cu-bridge/tools:${MACA_PATH}/tools/cu-bridge/bin:${PATH} +ENV LD_LIBRARY_PATH=/opt/mxdriver/lib:${MACA_PATH}/lib:${MACA_PATH}/mxgpu_llvm/lib:${MACA_PATH}/ompi/lib:${MACA_PATH}/ucx/lib:${LD_LIBRARY_PATH} + +# uv environment variables +ENV VIRTUAL_ENV=/opt/venv +ENV UV_INDEX_STRATEGY="unsafe-best-match" +ENV UV_HTTP_TIMEOUT=6000 +ENV UV_LINK_MODE=copy +ARG UV_EXTRA_INDEX_URL +ENV UV_EXTRA_INDEX_URL=${UV_EXTRA_INDEX_URL} +ARG UV_INDEX_URL +ENV UV_INDEX_URL=https://mirrors.aliyun.com/pypi/simple +ENV UV_TRUSTED_INDEX_HOST=mirrors.aliyun.com +ENV UV_OVERRIDE=/workspace/override.txt + +# vllm compile option +ENV VLLM_INSTALL_PUNICA_KERNELS=1 + +# AI version arguments +ARG PYTHON_VERSION +ARG VLLM_VERSION +ARG VLLM_METAX_VERSION +ARG MACA_VERSION +ARG MEGATRON_VERSION +ARG SWIFT_VERSION +ARG CU_BRIDGE_VERSION +ARG TE_VERSION + +WORKDIR /workspace +COPY override.txt /workspace/override.txt +COPY requirements_extra.txt /workspace/requirements_extra.txt + +RUN printf "[metax-centos]\n\ +name=Maca Driver Yum Repository\n\ +baseurl=https://repos.metax-tech.com/r/metax-driver-centos-$(uname -m)/\n\ +enabled=1\n\ +gpgcheck=0" > /etc/yum.repos.d/metax-driver-centos.repo + +RUN dnf -y install python3-pip hostname && \ + dnf clean all + +RUN python3 -m pip install uv -i $UV_INDEX_URL --trusted-host ${UV_TRUSTED_INDEX_HOST} && \ + uv venv /opt/venv --python=${PYTHON_VERSION} + +RUN python3 --version && \ + uv self version + +RUN yum makecache && yum install -y \ + unzip vim git openblas-devel make cmake \ + ninja-build gcc g++ procps-ng \ + libibverbs librdmacm libibumad \ + && yum clean all + +RUN git clone --depth 1 --branch ${SWIFT_VERSION} https://github.com/modelscope/ms-swift.git +RUN git clone --depth 1 --branch ${VLLM_METAX_VERSION} https://github.com/MetaX-MACA/vLLM-metax.git +RUN git clone --depth 1 --branch ${VLLM_VERSION} https://github.com/vllm-project/vllm.git +RUN git clone --depth 1 --branch ${MEGATRON_VERSION} https://github.com/NVIDIA/Megatron-LM.git + +# Step 1: install MACA SDK, Metax-Driver and cu-bridge +# Metax-Driver mainly contains vbios and kmd files, which are not needed in a container. +# Here we keep the mx-smi management tool. Kernel version mismatch errors are ignored. +RUN yum makecache && \ + yum install -y metax-driver-${MACA_VERSION}* mxgvm && \ + yum clean all && rm -rf /var/cache/yum /tmp/* + +RUN printf "[maca-sdk]\n\ +name=Maca Sdk Yum Repository\n\ +baseurl=https://repos.metax-tech.com/r/maca-sdk-rpm-$(uname -m)/\n\ +enabled=1\n\ +gpgcheck=0" > /etc/yum.repos.d/maca-sdk-rpm.repo + +RUN yum makecache && \ + yum install -y maca_sdk-${MACA_VERSION}* && \ + yum clean all && rm -rf /var/cache/yum /tmp/* + +RUN cd /tmp/ && \ + export MACA_PATH=/opt/maca && \ + curl -o ${CU_BRIDGE_VERSION}.zip -LsSf https://gitee.com/metax-maca/cu-bridge/repository/archive/${CU_BRIDGE_VERSION}.zip && \ + unzip ${CU_BRIDGE_VERSION}.zip && \ + mv cu-bridge-${CU_BRIDGE_VERSION} cu-bridge && \ + chmod 755 cu-bridge -Rf && \ + cd cu-bridge && \ + mkdir build && cd build && \ + cmake -DCMAKE_INSTALL_PREFIX=/opt/maca/tools/cu-bridge ../ && \ + make && make install + +# Step 2: trim unused MACA packages and install build prerequisites +RUN cd vLLM-metax && \ + uv pip install -r requirements/build.txt && \ + uv pip install build + +RUN yum makecache && yum install -y \ + gcc \ + binutils \ + procps-ng \ + libibverbs \ + librdmacm \ + libibumad \ + openblas \ + numactl-libs \ + && yum clean all && rm -rf /var/cache/yum /tmp/* + +# Step 3: install Metax python requirements +RUN cd vLLM-metax && \ + UV_HTTP_TIMEOUT=960 uv pip install -r requirements/maca.txt --trusted-host ${UV_TRUSTED_HOST} + +# Step 4: build vLLM with empty device to avoid CUDA dependency +RUN cd vllm && \ + python3 use_existing_torch.py && \ + uv pip install -r requirements/build.txt + +RUN cd vllm && \ + VLLM_TARGET_DEVICE=empty uv pip install -v . --no-build-isolation + +# Step 5: build vLLM-metax +RUN cd vLLM-metax && \ + uv pip install -r requirements/build.txt && \ + python3 -m build -w -n && \ + uv pip install dist/*.whl + +# Step 6: install Megatron-LM +RUN sed -i 's/nvcc/cucc/g' /workspace/Megatron-LM/megatron/legacy/fused_kernels/__init__.py && \ + cd Megatron-LM && \ + uv pip install . + +# Step 7: install transformer-engine +RUN uv pip install transformer_engine==${TE_VERSION} -i https://repos.metax-tech.com/r/maca-pypi/simple --trusted-host ${UV_TRUSTED_HOST} + +# Step 8: patch and install ms-swift v4.1.0 with Megatron extra dependencies +RUN sed -i '0,/^\(from \|import \)/{s//import vllm_metax.patch\n&/}' ms-swift/swift/__init__.py && \ + cd ms-swift && \ + uv pip install '.[megatron]' + +# Step 9: install optional runtime dependencies used by swift 4.1.0 +RUN uv pip install deepspeed -i https://repos.metax-tech.com/r/maca-pypi/simple --trusted-host ${UV_TRUSTED_HOST} +RUN uv pip install pip +RUN uv pip install -r requirements_extra.txt +RUN ln -sf ${CUDA_PATH}/bin/nvcc ${CUDA_PATH}/bin/cucc + +# vllm installation may bring in incompatible CUDA-only wheels. Remove them here. +RUN uv pip uninstall flashinfer-python cupy-cuda12x flash-linear-attention fla-core + +#################### FINAL IMAGE #################### diff --git a/docker/Metax/4.1/Dockerfile.with_metax_image b/docker/Metax/4.1/Dockerfile.with_metax_image new file mode 100644 index 00000000..296f5668 --- /dev/null +++ b/docker/Metax/4.1/Dockerfile.with_metax_image @@ -0,0 +1,73 @@ +ARG BUILD_BASE_IMAGE=mx-devops-acr-cn-shanghai.cr.volces.com/opensource/public-ai-release/maca/ms-swift:4.0.4-maca.ai3.5.3.5-torch2.8-py312-ubuntu22.04-amd64 +ARG PYTHON_VERSION=3.12 + +FROM ${BUILD_BASE_IMAGE} AS base + +# NOTE: +# This fast-build path inherits Python/Torch/TE from a prebuilt Metax release image. +# We keep the verified base image tag here instead of guessing a newer one. +# As a result, this path may lag behind the Megatron-SWIFT Quick Start recommendations. + +# may need passing a particular vllm version during build +ARG VLLM_VERSION +ARG VLLM_METAX_VERSION +ARG MEGATRON_VERSION +ARG SWIFT_VERSION + +ENV MACA_PATH=/opt/maca +ENV CUCC_CMAKE_ENTRY=2 +ENV CUDA_PATH=/root/cu-bridge/CUDA_DIR +ENV CUCC_PATH=${MACA_PATH}/tools/cu-bridge +ENV PATH=/opt/conda/bin:/opt/conda/condabin:${CUDA_PATH}/bin:${CUCC_PATH}/tools:${CUCC_PATH}/bin:${MACA_PATH}/bin:${PATH} +ENV LD_LIBRARY_PATH=${CUDA_PATH}/lib64:${MACA_PATH}/lib:${MACA_PATH}/mxgpu_llvm/lib:${LD_LIBRARY_PATH} + +WORKDIR /workspace +COPY requirements_extra.txt /workspace/requirements_extra.txt + +RUN echo $PATH +RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* + +# Initialize cu-bridge if it is not already prepared in the base image. +RUN if [ ! -d /root/cu-bridge ]; then \ + ${MACA_PATH}/tools/cu-bridge/tools/pre_make; \ + fi + +# Clone all GitHub sources while the external proxy is enabled. +RUN rm -rf /workspace/ms-swift /workspace/vLLM-metax /workspace/vllm /workspace/Megatron-LM +RUN git clone --depth 1 --branch ${SWIFT_VERSION} https://github.com/modelscope/ms-swift.git +RUN git clone --depth 1 --branch ${VLLM_METAX_VERSION} https://github.com/MetaX-MACA/vLLM-metax.git +RUN git clone --depth 1 --branch ${VLLM_VERSION} https://github.com/vllm-project/vllm.git +RUN git clone --depth 1 --branch ${MEGATRON_VERSION} https://github.com/NVIDIA/Megatron-LM.git + +# install cmake +RUN pip install cmake ninja + +# Step 1: build original vLLM for torch setup +RUN cd vllm && \ + python3 use_existing_torch.py && \ + pip install -r requirements/build.txt + +# Step 2: build vLLM with empty device to avoid CUDA dependency +RUN cd vllm && \ + VLLM_TARGET_DEVICE=empty pip install -v . --no-build-isolation + +# Step 3: build vLLM-metax +RUN cd vLLM-metax && \ + python3 use_existing_metax.py && \ + pip install -r requirements/build.txt && \ + python3 -m build -w -n && \ + pip install dist/*.whl + +# Step 4: patch and install Megatron-LM +RUN sed -i 's/nvcc/cucc/g' /workspace/Megatron-LM/megatron/legacy/fused_kernels/__init__.py && \ + cd /workspace/Megatron-LM && \ + pip install . + +# Step 5: patch and install ms-swift v4.1.0 with its Megatron extra +RUN sed -i '0,/^\(from \|import \)/{s//import vllm_metax.patch\n&/}' ms-swift/swift/__init__.py && \ + cd ms-swift && \ + pip install "transformers<5.4.0" && \ + pip install '.[megatron]' && \ + pip install -r /workspace/requirements_extra.txt + +CMD ["bash"] diff --git a/docker/Metax/4.1/build.sh b/docker/Metax/4.1/build.sh new file mode 100644 index 00000000..6c112b53 --- /dev/null +++ b/docker/Metax/4.1/build.sh @@ -0,0 +1,13 @@ +docker build \ + --network host \ + -f Dockerfile.metax \ + -t swift:v4.1.0 \ + --build-arg VLLM_VERSION=v0.17.1 \ + --build-arg VLLM_METAX_VERSION=v0.17.0 \ + --build-arg MACA_VERSION=3.5.3 \ + --build-arg MEGATRON_VERSION=core_v0.16.0 \ + --build-arg SWIFT_VERSION=v4.1.0 \ + --build-arg TE_VERSION=2.8.0 \ + --build-arg CU_BRIDGE_VERSION=3.5.3 \ + --no-cache \ + . diff --git a/docker/Metax/4.1/build_from_metax_image.sh b/docker/Metax/4.1/build_from_metax_image.sh new file mode 100644 index 00000000..3c2fb454 --- /dev/null +++ b/docker/Metax/4.1/build_from_metax_image.sh @@ -0,0 +1,10 @@ +docker build \ + --network host \ + -f Dockerfile.with_metax_image \ + -t swift:v4.1.0 \ + --build-arg VLLM_VERSION=v0.17.1 \ + --build-arg VLLM_METAX_VERSION=v0.17.0 \ + --build-arg MEGATRON_VERSION=core_v0.16.0 \ + --build-arg SWIFT_VERSION=v4.1.0 \ + --no-cache \ + . diff --git a/docker/Metax/4.1/override.txt b/docker/Metax/4.1/override.txt new file mode 100644 index 00000000..4a3f3b38 --- /dev/null +++ b/docker/Metax/4.1/override.txt @@ -0,0 +1,5 @@ +setuptools>=77.0.3,<80 +datasets>=3.0,<4.0 +flash-linear-attention +mcoplib +transformers<5.4.0 diff --git a/docker/Metax/4.1/requirements_extra.txt b/docker/Metax/4.1/requirements_extra.txt new file mode 100644 index 00000000..3480e1d9 --- /dev/null +++ b/docker/Metax/4.1/requirements_extra.txt @@ -0,0 +1,14 @@ +decord +diffusers==0.35.2 +evalscope>=1.0.0 +evalscope[opencompass] +evalscope[vlmeval] +keye_vl_utils>=1.5.2 +librosa +mpi4py +optimum==1.27.0 +pytorchvideo +qwen_omni_utils>=0.0.9 +qwen_vl_utils==0.0.14 +soundfile +timm diff --git a/docker/Metax/4.1/swift_building_instructions.md b/docker/Metax/4.1/swift_building_instructions.md new file mode 100644 index 00000000..c7a33091 --- /dev/null +++ b/docker/Metax/4.1/swift_building_instructions.md @@ -0,0 +1,52 @@ +# 1. Build swift 4.1 image from a UBI9 base image + Full build from a minimal base image, using a venv virtual environment. + +## 1.1. Build + ``` bash + bash build.sh + ``` + +## 1.2. Run a container + ``` bash + docker run -d -it --net=host --uts=host --ipc=host --privileged=true --group-add video \ + --shm-size 100gb --ulimit memlock=-1 \ + --security-opt seccomp=unconfined --security-opt apparmor=unconfined \ + --device=/dev/dri --device=/dev/mxcd \ + --name base_image \ + ${IMAGE_ID} bash + ``` + +## 1.3. Activate the venv environment + ``` bash + source /opt/venv/bin/activate + ``` + +## 1.4. Run swift examples + ``` bash + cd /workspace/ms-swift + bash examples/train/full/train.sh + ``` + +# 2. Build swift 4.1 image from a Metax release image + Faster build based on the pre-built Metax release image. + +## 2.1. Build + ``` bash + bash build_from_metax_image.sh + ``` + +## 2.2. Run a container + ``` bash + docker run -d -it --net=host --uts=host --ipc=host --privileged=true --group-add video \ + --shm-size 100gb --ulimit memlock=-1 \ + --security-opt seccomp=unconfined --security-opt apparmor=unconfined \ + --device=/dev/dri --device=/dev/mxcd \ + --name base_image \ + ${IMAGE_ID} bash + ``` + +## 2.3. Run swift examples + ``` bash + cd /workspace/ms-swift + bash examples/train/full/train.sh + ``` From c15c5261e1c328684f6559cf472f2fd13c6aeacb Mon Sep 17 00:00:00 2001 From: "Xingjun.Wang" Date: Fri, 8 May 2026 15:13:52 +0800 Subject: [PATCH 05/23] [Fix] Fix dataset preview args, private streaming auth, and download retries (#1700) --- .../msdatasets/utils/hf_datasets_util.py | 50 +++++++++++++++++-- modelscope/msdatasets/utils/hf_file_utils.py | 34 +++++++++++-- 2 files changed, 75 insertions(+), 9 deletions(-) diff --git a/modelscope/msdatasets/utils/hf_datasets_util.py b/modelscope/msdatasets/utils/hf_datasets_util.py index f422e064..40167606 100644 --- a/modelscope/msdatasets/utils/hf_datasets_util.py +++ b/modelscope/msdatasets/utils/hf_datasets_util.py @@ -12,6 +12,7 @@ Sub-modules: _module_factories – dataset module factory functions & data-file resolution """ import contextlib +import inspect import os import warnings from dataclasses import fields @@ -407,10 +408,26 @@ def _get_paths_info( # =================================================================== -# HfFileSystem patch (_hf_fs_open) +# HfFileSystem patches (_hf_fs_open, _hf_fs_init) # =================================================================== _hf_fs_open_original = None +_hf_fs_init_original = None + + +def _hf_fs_init_with_cookie(self, *args, endpoint=None, token=None, **kwargs): + """HfFileSystem.__init__ wrapper that injects ModelScope cookie auth. + + ModelScope's /resolve/ endpoint authenticates via `m_session_id` cookie + rather than the `Authorization: Bearer` header used by HuggingFace Hub. + This wrapper ensures the cookie is included in all subsequent HTTP + requests made by the HfFileSystem instance. + """ + _hf_fs_init_original(self, *args, endpoint=endpoint, token=token, **kwargs) + if token and isinstance(token, str): + if not hasattr(self._api, 'headers') or self._api.headers is None: + self._api.headers = {} + self._api.headers['Cookie'] = f'm_session_id={token}' def _hf_fs_open(self, path, mode='rb', **kwargs): @@ -769,6 +786,24 @@ class DatasetsWrapperHF: builder_cls = get_dataset_builder_class( dataset_module, dataset_name=dataset_name) + _config_cls = builder_cls.BUILDER_CONFIG_CLASS + if hasattr(_config_cls, '__dataclass_fields__'): + _valid_fields = set(_config_cls.__dataclass_fields__.keys()) + # Also preserve parameters accepted by the builder's + # __init__ (e.g. writer_batch_size, base_path, repo_id) + # so they are not inadvertently stripped. + try: + _init_params = set( + inspect.signature(builder_cls.__init__).parameters.keys() + ) + except (ValueError, TypeError): + _init_params = set() + _valid_fields = _valid_fields | _init_params + config_kwargs = { + k: v for k, v in config_kwargs.items() + if k in _valid_fields + } + builder_instance: DatasetBuilder = builder_cls( cache_dir=cache_dir, dataset_name=dataset_name, @@ -1032,7 +1067,7 @@ def load_dataset_with_ctx(*args, **kwargs): non-streaming mode) or kept alive (for streaming mode, where lazy iteration needs the patches to remain active). """ - global _hf_fs_open_original + global _hf_fs_open_original, _hf_fs_init_original # Save originals hf_endpoint_origin = config.HF_ENDPOINT @@ -1048,6 +1083,7 @@ def load_dataset_with_ctx(*args, **kwargs): HubDatasetModuleFactoryWithScript.get_module if _HAS_SCRIPT_LOADING else None) generate_from_dict_origin = features.generate_from_dict hf_fs_open_origin = HfFileSystem._open + hf_fs_init_origin = HfFileSystem.__init__ # Apply patches config.HF_ENDPOINT = get_endpoint() @@ -1066,6 +1102,8 @@ def load_dataset_with_ctx(*args, **kwargs): features.generate_from_dict = generate_from_dict_ms _hf_fs_open_original = hf_fs_open_origin HfFileSystem._open = _hf_fs_open + _hf_fs_init_original = hf_fs_init_origin + HfFileSystem.__init__ = _hf_fs_init_with_cookie streaming = kwargs.get('streaming', False) @@ -1076,10 +1114,12 @@ def load_dataset_with_ctx(*args, **kwargs): _repo_tree_cache.clear() HubApi._dataset_id_type_cache.clear() - HfFileSystem._open = hf_fs_open_origin - _hf_fs_open_original = None - if not streaming: + HfFileSystem._open = hf_fs_open_origin + _hf_fs_open_original = None + HfFileSystem.__init__ = hf_fs_init_origin + _hf_fs_init_original = None + config.HF_ENDPOINT = hf_endpoint_origin file_utils.get_from_cache = get_from_cache_origin features.generate_from_dict = generate_from_dict_origin diff --git a/modelscope/msdatasets/utils/hf_file_utils.py b/modelscope/msdatasets/utils/hf_file_utils.py index b8a327f6..8c30a359 100644 --- a/modelscope/msdatasets/utils/hf_file_utils.py +++ b/modelscope/msdatasets/utils/hf_file_utils.py @@ -53,7 +53,7 @@ def _request_with_retry_ms( url: str, max_retries: int = 2, base_wait_time: float = 0.5, - max_wait_time: float = 8, + max_wait_time: float = 3, timeout: float = 10.0, **params, ) -> requests.Response: @@ -73,14 +73,35 @@ def _request_with_retry_ms( """ tries, success = 0, False response = None + range_header = (params.get('headers') or {}).get('Range', '') while not success: tries += 1 try: + logger.debug( + '[MS_DOWNLOAD] _request_with_retry_ms sending request: ' + 'method=%s, url=%s, timeout=%s, Range=%s', + method, url, timeout, range_header or 'N/A', + ) + t0 = time.perf_counter() response = requests.request(method=method.upper(), url=url, timeout=timeout, **params) + elapsed = time.perf_counter() - t0 + logger.debug( + '[MS_DOWNLOAD] _request_with_retry_ms response: ' + 'status=%s, content_length=%s, elapsed=%.3fs, url=%s', + response.status_code, + response.headers.get('Content-Length', 'N/A'), + elapsed, + url, + ) success = True - except (requests.exceptions.ConnectTimeout, - requests.exceptions.ConnectionError, - requests.exceptions.ReadTimeout) as err: + except (requests.exceptions.ReadTimeout, + requests.exceptions.ConnectTimeout, + requests.exceptions.ConnectionError) as err: + logger.error( + '[MS_DOWNLOAD] _request_with_retry_ms %s: ' + 'method=%s, url=%s, timeout=%s, error=%s', + type(err).__name__, method, url, timeout, err, + ) if tries > max_retries: raise err else: @@ -111,6 +132,10 @@ def http_head_ms( def http_get_ms( url, temp_file, proxies=None, resume_size=0, headers=None, cookies=None, timeout=300.0, max_retries=3, desc=None ) -> Optional[requests.Response]: + logger.debug( + '[MS_DOWNLOAD] http_get_ms entry: url=%s, timeout=%s, resume_size=%s', + url, timeout, resume_size, + ) headers = dict(headers) if headers is not None else {} headers['user-agent'] = get_datasets_user_agent_ms(user_agent=headers.get('user-agent')) if resume_size > 0: @@ -323,6 +348,7 @@ def get_from_cache_ms( if scheme not in ('http', 'https'): fsspec_get(url, temp_file, storage_options=storage_options, desc=download_desc) else: + logger.info('[MS_DOWNLOAD] get_from_cache_ms downloading: url=%s', url) http_get_ms( url, temp_file=temp_file, From 2da1d9491395e49b42561e591230a8fa162d58de Mon Sep 17 00:00:00 2001 From: "Xingjun.Wang" Date: Fri, 8 May 2026 15:14:08 +0800 Subject: [PATCH 06/23] fix upload cache ignore (#1709) --- modelscope/hub/api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 4e700ad7..2a0adf54 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -2753,8 +2753,9 @@ class HubApi: ) commit_description = commit_description or 'Uploading files' - # Exclude internal cache/checkpoint files from upload - _internal_ignore = [UPLOAD_HASH_CACHE_FILE, _LEGACY_PROGRESS_FILE] + # Exclude internal cache/checkpoint files from upload at any directory depth + _internal_files = [UPLOAD_HASH_CACHE_FILE, _LEGACY_PROGRESS_FILE] + _internal_ignore = [p for f in _internal_files for p in (f, f'*/{f}')] if ignore_patterns is None: ignore_patterns = _internal_ignore elif isinstance(ignore_patterns, str): From c2b8d9f020f9c7f0b9f4a1bc6a1d9ce2c404e1b2 Mon Sep 17 00:00:00 2001 From: tastelikefeet <58414341+tastelikefeet@users.noreply.github.com> Date: Sat, 9 May 2026 14:44:09 +0800 Subject: [PATCH 07/23] Fix plugin rce (#1703) --- modelscope/pipelines/builder.py | 23 ++++++++++++++++++++--- modelscope/trainers/builder.py | 26 +++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/modelscope/pipelines/builder.py b/modelscope/pipelines/builder.py index 5b2d1481..fcc58701 100644 --- a/modelscope/pipelines/builder.py +++ b/modelscope/pipelines/builder.py @@ -79,6 +79,7 @@ def pipeline(task: str = None, device: str = None, model_revision: Optional[str] = DEFAULT_MODEL_REVISION, ignore_file_pattern: List[str] = None, + trust_remote_code: bool = False, **kwargs) -> Pipeline: """ Factory method to build an obj:`Pipeline`. @@ -95,6 +96,8 @@ def pipeline(task: str = None, device (str, optional): whether to use gpu or cpu is used to do inference. ignore_file_pattern(`str` or `List`, *optional*, default to `None`): Any file pattern to be ignored in downloading, like exact file names or file extensions. + trust_remote_code (bool, optional): Whether to allow execution of remote code or + plugins declared in the model configuration. Defaults to False. Return: pipeline (obj:`Pipeline`): pipeline object for certain task. @@ -155,9 +158,23 @@ def pipeline(task: str = None, third_party=third_party, ignore_file_pattern=ignore_file_pattern) - register_plugins_repo(cfg.safe_get('plugins')) - register_modelhub_repo(model, - cfg.get('allow_remote', False)) + if cfg: + plugins = cfg.safe_get('plugins') + allow_remote = cfg.get('allow_remote', False) + if (plugins or allow_remote) and not trust_remote_code: + raise RuntimeError( + 'Detected plugins or allow_remote field in the model ' + 'configuration file, but trust_remote_code=True was not ' + 'explicitly set.\n' + 'To prevent potential execution of malicious code, loading ' + 'has been refused.\n' + 'If you trust this model repository, please pass ' + 'trust_remote_code=True to pipeline().') + register_plugins_repo(plugins) + model_dir = model if isinstance(model, + str) else model[0] + register_modelhub_repo( + model_dir, trust_remote_code and allow_remote) if pipeline_name: pipeline_props = {'type': pipeline_name} diff --git a/modelscope/trainers/builder.py b/modelscope/trainers/builder.py index 8d1847f6..eccd72fc 100644 --- a/modelscope/trainers/builder.py +++ b/modelscope/trainers/builder.py @@ -19,10 +19,15 @@ def build_trainer(name: str = Trainers.default, default_args: dict = None): name (str, optional): Trainer name, if None, default trainer will be used. default_args (dict, optional): Default initialization arguments. + If ``trust_remote_code`` key is set to True in default_args, + remote code and plugins declared in the model configuration + will be allowed to execute. """ cfg = dict(type=name) + default_args = default_args or {} model = default_args.get('model', None) model_revision = default_args.get('model_revision', DEFAULT_MODEL_REVISION) + trust_remote_code = default_args.get('trust_remote_code', False) if isinstance(model, str) \ or (isinstance(model, list) and isinstance(model[0], str)): @@ -33,7 +38,22 @@ def build_trainer(name: str = Trainers.default, default_args: dict = None): model, str) else read_config( model[0], revision=model_revision) model_dir = normalize_model_input(model, model_revision) - register_plugins_repo(configuration.safe_get('plugins')) - register_modelhub_repo(model_dir, - configuration.get('allow_remote', False)) + if configuration: + plugins = configuration.safe_get('plugins') + allow_remote = configuration.get('allow_remote', False) + if (plugins or allow_remote) and not trust_remote_code: + raise RuntimeError( + 'Detected plugins or allow_remote field in the model ' + 'configuration file, but trust_remote_code=True was ' + 'not explicitly set.\n' + 'To prevent potential execution of malicious code, ' + 'loading has been refused.\n' + 'If you trust this model repository, please pass ' + 'trust_remote_code=True in default_args to ' + 'build_trainer().') + register_plugins_repo(plugins) + model_dir_str = model_dir if isinstance(model_dir, + str) else model_dir[0] + register_modelhub_repo(model_dir_str, trust_remote_code + and allow_remote) return build_from_cfg(cfg, TRAINERS, default_args=default_args) From a7072b52b844e2b6a5450859b3ac3a39003d5fe9 Mon Sep 17 00:00:00 2001 From: Jintao Huang Date: Sat, 9 May 2026 14:59:04 +0800 Subject: [PATCH 08/23] bump version --- modelscope/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modelscope/version.py b/modelscope/version.py index 33e5ef09..76be6b7e 100644 --- a/modelscope/version.py +++ b/modelscope/version.py @@ -1,5 +1,5 @@ # Make sure to modify __release_datetime__ to release time when making official release. -__version__ = '2.0.0+main' +__version__ = '1.37.0' # default release datetime for branches under active development is set # to be a time far-far-away-into-the-future -__release_datetime__ = '2099-09-06 00:00:00' +__release_datetime__ = '2026-05-09 23:59:59' From 16991606f4140c0722d12509aa684193ccf15713 Mon Sep 17 00:00:00 2001 From: Jintao Huang Date: Sun, 10 May 2026 18:00:21 +0800 Subject: [PATCH 09/23] update docker --- .github/workflows/docker-image.yml | 4 ++-- docker/Dockerfile.ubuntu | 8 ++++---- docker/build_image.py | 21 ++++++++++++--------- docker/install.sh | 5 ----- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 0da55ffc..f5c576da 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -27,9 +27,9 @@ on: description: 'Other params in --xxx xxx' required: false python_version: - description: 'Python version to use, default is 3.11.11' + description: 'Python version to use, default is 3.12.13' required: false - default: '3.11.11' + default: '3.12.13' run-name: Docker-${{ inputs.modelscope_branch }}-${{ inputs.image_type }}-${{ inputs.workflow_name }}-${{ inputs.python_version }}-by-@${{ github.actor }} diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu index e74c3374..d2d356a3 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.ubuntu @@ -65,14 +65,14 @@ RUN bash /tmp/install.sh {version_args} && \ RUN if [ "$IMAGE_TYPE" = "gpu" ]; then \ - pip install --no-cache-dir math_verify "gradio<5.33" "deepspeed<0.18" ray -U && \ - pip install --no-cache-dir liger_kernel wandb swanlab nvitop pre-commit "transformers" "trl<0.25" "peft<0.18" huggingface-hub -U && \ + pip install --no-cache-dir math_verify "gradio<5.33" "deepspeed<0.19" ray -U && \ + pip install --no-cache-dir mcore-bridge -i https://pypi.org/simple/ -U && \ + pip install --no-cache-dir liger_kernel wandb swanlab nvitop pre-commit "transformers<5.9" "trl<1.0" "peft<0.20" huggingface-hub -U && \ pip install --no-cache-dir --no-build-isolation transformer_engine[pytorch]; \ cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone https://github.com/NVIDIA/apex && \ cd apex && pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" ./ && \ cd / && rm -fr /tmp/apex && pip cache purge; \ - pip install --no-cache-dir "megatron-core==0.15.*"; \ - pip uninstall autoawq -y; \ + pip install --no-cache-dir "megatron-core==0.16.*" -U; \ elif [ "$IMAGE_TYPE" = "cpu" ]; then \ pip install --no-cache-dir huggingface-hub transformers peft diffusers -U; \ else \ diff --git a/docker/build_image.py b/docker/build_image.py index a80879b0..325b3dd9 100644 --- a/docker/build_image.py +++ b/docker/build_image.py @@ -339,6 +339,15 @@ class StableCPUImageBuilder(Builder): class StableGPUImageBuilder(Builder): """Dependencies will be stable versions""" + def init_args(self, args: Any) -> Any: + if not args.torch_version: + args.torch_version = '2.10.0' + args.torchaudio_version = '2.10.0' + args.torchvision_version = '0.25.0' + if not args.vllm_version: + args.vllm_version = '0.19.1' + return super().init_args(args) + def generate_dockerfile(self) -> str: meta_file = './docker/install.sh' with open('docker/Dockerfile.extra_install', 'r') as f: @@ -400,11 +409,6 @@ RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy class LatestGPUImageBuilder(StableGPUImageBuilder): """Dependencies will be latest versions""" - def init_args(self, args: Any) -> Any: - if not args.vllm_version: - args.vllm_version = '0.16.0' - return super().init_args(args) - def generate_dockerfile(self) -> str: meta_file = './docker/install.sh' with open('docker/Dockerfile.extra_install', 'r') as f: @@ -417,8 +421,7 @@ RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy version_args = ( f'{self.args.torch_version} {self.args.torchvision_version} {self.args.torchaudio_version} ' f'{self.args.vllm_version} {self.args.lmdeploy_version} {self.args.autogptq_version} ' - f'{self.args.optimum_version}' - f'{self.args.flashattn_version}') + f'{self.args.flashattn_version} {self.args.optimum_version}') with open('docker/Dockerfile.ubuntu', 'r') as f: content = f.read() content = content.replace('{base_image}', self.args.base_image) @@ -426,7 +429,7 @@ RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy content = content.replace('{meta_file}', meta_file) content = content.replace('{version_args}', version_args) content = content.replace('{cur_time}', formatted_time) - content = content.replace('{install_ms_deps}', 'True') + content = content.replace('{install_ms_deps}', 'False') content = content.replace('{image_type}', 'gpu') content = content.replace('{torch_version}', self.args.torch_version) @@ -533,7 +536,7 @@ RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy parser = argparse.ArgumentParser() parser.add_argument('--base_image', type=str, default=None) parser.add_argument('--image_type', type=str) -parser.add_argument('--python_version', type=str, default='3.11.11') +parser.add_argument('--python_version', type=str, default='3.12.13') parser.add_argument('--ubuntu_version', type=str, default='22.04') parser.add_argument('--torch_version', type=str, default=None) parser.add_argument('--torchvision_version', type=str, default=None) diff --git a/docker/install.sh b/docker/install.sh index e27e30db..fa5cf927 100644 --- a/docker/install.sh +++ b/docker/install.sh @@ -7,14 +7,11 @@ vllm_version=${4:-0.6.0} lmdeploy_version=${5:-0.6.1} autogptq_version=${6:-0.7.1} flashattn_version=${7:-2.7.1.post4} -optimum_version=${8:-2.0.0} pip uninstall -y torch torchvision torchaudio pip install --no-cache-dir torch==$torch_version torchvision==$torchvision_version torchaudio==$torchaudio_version -pip install --no-cache-dir torch==$torch_version torchvision==$torchvision_version torchaudio==$torchaudio_version - pip install --no-cache-dir tiktoken transformers_stream_generator bitsandbytes deepspeed torchmetrics decord optimum openai-whisper # pip install https://github.com/Dao-AILab/flash-attention/releases/download/v2.6.3/flash_attn-2.6.3+cu123torch2.4cxx11abiTRUE-cp310-cp310-linux_x86_64.whl @@ -24,8 +21,6 @@ MAX_JOBS=16 pip install --no-cache-dir flash_attn==$flashattn_version --no-build pip install --no-cache-dir triton -U && pip cache purge -pip install --no-cache-dir optimum==$optimum_version - if [[ "$(printf '%s\n' "0.6.0" "$vllm_version" | sort -V | head -n1)" = "0.6.0" ]]; then # vllm_version is >= 0.6.0 pip install --no-cache-dir vllm==$vllm_version && pip cache purge From 263a3ef0b2f5cd1c17797ecf7afb48d5acae83fa Mon Sep 17 00:00:00 2001 From: Jintao Huang Date: Sun, 10 May 2026 22:58:21 +0800 Subject: [PATCH 10/23] fix --- docker/build_image.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docker/build_image.py b/docker/build_image.py index 325b3dd9..daef43e2 100644 --- a/docker/build_image.py +++ b/docker/build_image.py @@ -2,6 +2,7 @@ import argparse import os import platform import subprocess +from copy import copy from datetime import datetime from typing import Any @@ -25,9 +26,9 @@ class Builder: # A mirrored image of nvidia/cuda:12.4.0-devel-ubuntu22.04 args.base_image = 'nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04' if not args.torch_version: - args.torch_version = '2.9.1' - args.torchaudio_version = '2.9.1' - args.torchvision_version = '0.24.1' + args.torch_version = '2.10.0' + args.torchaudio_version = '2.10.0' + args.torchvision_version = '0.25.0' if not args.optimum_version: args.optimum_version = '2.0.0' if not args.tf_version: @@ -35,7 +36,7 @@ class Builder: if not args.cuda_version: args.cuda_version = '12.8.1' if not args.vllm_version: - args.vllm_version = '0.15.1' + args.vllm_version = '0.19.1' if not args.lmdeploy_version: args.lmdeploy_version = '0.11.0' if not args.autogptq_version: @@ -571,4 +572,5 @@ else: raise ValueError(f'Unsupported image_type: {args.image_type}') for builder in builder_cls: + args = copy(args) builder(args, args.dry_run)() From b7d47748ed9ccbb4f80e0d5112121b3f395510fa Mon Sep 17 00:00:00 2001 From: tastelikefeet <58414341+tastelikefeet@users.noreply.github.com> Date: Tue, 12 May 2026 17:40:06 +0800 Subject: [PATCH 11/23] fix plugin (#1712) --- modelscope/pipelines/builder.py | 6 ++++-- modelscope/trainers/builder.py | 7 ++++--- modelscope/utils/plugins.py | 13 +++++++++++++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/modelscope/pipelines/builder.py b/modelscope/pipelines/builder.py index fcc58701..b0095b73 100644 --- a/modelscope/pipelines/builder.py +++ b/modelscope/pipelines/builder.py @@ -12,7 +12,8 @@ from modelscope.utils.constant import (DEFAULT_MODEL_REVISION, Invoke, Tasks, from modelscope.utils.hub import read_config from modelscope.utils.import_utils import is_transformers_available from modelscope.utils.logger import get_logger -from modelscope.utils.plugins import (register_modelhub_repo, +from modelscope.utils.plugins import (filter_plugin_in_whitelist, + register_modelhub_repo, register_plugins_repo) from modelscope.utils.registry import Registry, build_from_cfg from modelscope.utils.task_utils import is_embedding_task @@ -161,7 +162,8 @@ def pipeline(task: str = None, if cfg: plugins = cfg.safe_get('plugins') allow_remote = cfg.get('allow_remote', False) - if (plugins or allow_remote) and not trust_remote_code: + if (filter_plugin_in_whitelist(plugins) + or allow_remote) and not trust_remote_code: raise RuntimeError( 'Detected plugins or allow_remote field in the model ' 'configuration file, but trust_remote_code=True was not ' diff --git a/modelscope/trainers/builder.py b/modelscope/trainers/builder.py index eccd72fc..2a04343c 100644 --- a/modelscope/trainers/builder.py +++ b/modelscope/trainers/builder.py @@ -2,10 +2,10 @@ from modelscope.metainfo import Trainers from modelscope.pipelines.builder import normalize_model_input from modelscope.pipelines.util import is_official_hub_path -from modelscope.utils.config import check_config from modelscope.utils.constant import DEFAULT_MODEL_REVISION from modelscope.utils.hub import read_config -from modelscope.utils.plugins import (register_modelhub_repo, +from modelscope.utils.plugins import (filter_plugin_in_whitelist, + register_modelhub_repo, register_plugins_repo) from modelscope.utils.registry import Registry, build_from_cfg @@ -41,7 +41,8 @@ def build_trainer(name: str = Trainers.default, default_args: dict = None): if configuration: plugins = configuration.safe_get('plugins') allow_remote = configuration.get('allow_remote', False) - if (plugins or allow_remote) and not trust_remote_code: + if (filter_plugin_in_whitelist(plugins) + or allow_remote) and not trust_remote_code: raise RuntimeError( 'Detected plugins or allow_remote field in the model ' 'configuration file, but trust_remote_code=True was ' diff --git a/modelscope/utils/plugins.py b/modelscope/utils/plugins.py index c4957202..fa4cb877 100644 --- a/modelscope/utils/plugins.py +++ b/modelscope/utils/plugins.py @@ -8,6 +8,7 @@ import importlib import importlib.metadata import os import pkgutil +import re import shutil import subprocess import sys @@ -46,6 +47,18 @@ OFFICIAL_PLUGINS = [ LOCAL_PLUGINS_FILENAME = '.modelscope_plugins' GLOBAL_PLUGINS_FILENAME = os.path.join(Path.home(), '.modelscope', 'plugins') DEFAULT_PLUGINS = [] +PLUGIN_WHITE_LIST = ['pai-easycv'] + + +def filter_plugin_in_whitelist(plugins): + if not plugins: + return plugins + if isinstance(plugins, str): + plugins = [plugins] + return [ + plugin for plugin in plugins if re.split(r'[><=!~]', plugin.strip()) + [0].strip() not in PLUGIN_WHITE_LIST + ] @contextmanager From d77bc8f83b4d6fc6bc33c1fbf91c9c304e23c806 Mon Sep 17 00:00:00 2001 From: tastelikefeet <58414341+tastelikefeet@users.noreply.github.com> Date: Tue, 12 May 2026 19:16:13 +0800 Subject: [PATCH 12/23] fix trainer + trust_remote_code (#1713) --- modelscope/models/base/base_model.py | 5 +++-- modelscope/trainers/trainer.py | 6 +++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/modelscope/models/base/base_model.py b/modelscope/models/base/base_model.py index e19227f0..0a693de0 100644 --- a/modelscope/models/base/base_model.py +++ b/modelscope/models/base/base_model.py @@ -14,7 +14,8 @@ from modelscope.utils.config import Config, ConfigDict from modelscope.utils.constant import DEFAULT_MODEL_REVISION, Invoke, ModelFile from modelscope.utils.device import verify_device from modelscope.utils.logger import get_logger -from modelscope.utils.plugins import (register_modelhub_repo, +from modelscope.utils.plugins import (filter_plugin_in_whitelist, + register_modelhub_repo, register_plugins_repo) logger = get_logger() @@ -190,7 +191,7 @@ class Model(ABC): # Security check: Only allow execution of remote code or plugins if trust_remote_code is True plugins = cfg.safe_get('plugins') - if plugins and not trust_remote_code: + if filter_plugin_in_whitelist(plugins) and not trust_remote_code: raise RuntimeError( 'Detected plugins field in the model configuration file, but ' 'trust_remote_code=True was not explicitly set.\n' diff --git a/modelscope/trainers/trainer.py b/modelscope/trainers/trainer.py index 25f948bc..ca11cad3 100644 --- a/modelscope/trainers/trainer.py +++ b/modelscope/trainers/trainer.py @@ -135,6 +135,7 @@ class EpochBasedTrainer(BaseTrainer): self._inner_iter = 0 self._stop_training = False self._compile = kwargs.get('compile', False) + self.trust_remote_code = kwargs.get('trust_remote_code', False) self.train_dataloader = None self.eval_dataloader = None @@ -814,7 +815,10 @@ class EpochBasedTrainer(BaseTrainer): override this method in a subclass. """ - model = Model.from_pretrained(self.model_dir, cfg_dict=self.cfg) + model = Model.from_pretrained( + self.model_dir, + cfg_dict=self.cfg, + trust_remote_code=self.trust_remote_code) if not isinstance(model, nn.Module) and hasattr(model, 'model'): return model.model elif isinstance(model, nn.Module): From a56e2ce08fc5e13dd94d73ce96b5f80ee1ed03de Mon Sep 17 00:00:00 2001 From: "Xingjun.Wang" Date: Wed, 13 May 2026 17:46:07 +0800 Subject: [PATCH 13/23] [Fix] fix invalid commit (#1715) --- modelscope/hub/api.py | 81 +++++++++++++++++++++++++++++++------------ 1 file changed, 59 insertions(+), 22 deletions(-) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 2a0adf54..2de69a44 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -2316,6 +2316,18 @@ class HubApi: commit_message=commit_message, ) + # Guard: skip sending empty commits (no effective file changes) + if not payload['actions']: + logger.info( + 'Commit skipped: no effective actions in payload ' + '(all files already exist).') + return CommitInfo( + commit_url='', + commit_message=commit_message, + commit_description=commit_description or '', + oid='no-op', + ) + response = self.session.post( url, headers=self.builder_headers(self.headers), @@ -2958,18 +2970,27 @@ class HubApi: except Exception as e: logger.error( f'Batch {batch_idx + 1}/{num_batches} commit failed: {e}') - for r in results: - tracker.mark_failed( - r['file_path_in_repo'], r['file_mtime'], - r['file_size_on_disk'], - error_type='commit_failed') - # Recover uploaded files to retry queue - for r in results: - total_failed_files.append( - ((r['file_path_in_repo'], r['file_path']), e)) - logger.warning( - f'Batch {batch_idx + 1}/{num_batches}: ' - f'{len(results)} uploaded file(s) recovered to retry queue.') + category = classify_error(e) + if not category.is_retryable: + # Permanent error: mark files as failed, do not retry + for r in results: + tracker.mark_failed( + r['file_path_in_repo'], r['file_mtime'], + r['file_size_on_disk'], + error_type='commit_' + category.value) + logger.error( + f'Batch {batch_idx + 1}/{num_batches}: ' + f'permanent failure ({category.value}), ' + f'{len(results)} file(s) will not be retried.') + else: + # Transient error: recover to retry queue + for r in results: + total_failed_files.append( + ((r['file_path_in_repo'], r['file_path']), e)) + logger.warning( + f'Batch {batch_idx + 1}/{num_batches}: ' + f'{len(results)} file(s) recovered to retry queue ' + f'(error_category={category.value}).') finally: tracker.save() @@ -3031,10 +3052,19 @@ class HubApi: except Exception as e: logger.error( f' Retry round {retry_round + 1} commit failed: {e}') - for result in retry_successes: - retry_failures.append( - ((result['file_path_in_repo'], - result.get('file_path', '')), e)) + category = classify_error(e) + if not category.is_retryable: + for result in retry_successes: + tracker.mark_failed( + result['file_path_in_repo'], + result['file_mtime'], + result['file_size_on_disk'], + error_type='commit_' + category.value) + else: + for result in retry_successes: + retry_failures.append( + ((result['file_path_in_repo'], + result.get('file_path', '')), e)) total_failed_files = retry_failures # Final tracker save @@ -3254,7 +3284,7 @@ class HubApi: repo_type=repo_type, revision=revision) commit_infos.append(commit_info) - # Mark committed + # Mark committed only after successful commit self._track_committed_batch(tracker, batch) logger.info( f'[ReAct] {round_name}: ' @@ -3262,11 +3292,18 @@ class HubApi: except Exception as e: logger.error( f'[ReAct] {round_name} commit failed: {e}') - # Recover uploaded files back to failures - for r in batch: - round_failures.append( - ((r['file_path_in_repo'], - r['file_path']), e)) + category = classify_error(e) + if not category.is_retryable: + for r in batch: + tracker.mark_failed( + r['file_path_in_repo'], r['file_mtime'], + r['file_size_on_disk'], + error_type='commit_' + category.value) + else: + for r in batch: + round_failures.append( + ((r['file_path_in_repo'], + r['file_path']), e)) # OBSERVE: classify new failures, enforce per-file retry limit new_retryable = [] From d968a2cf8726ec3208a161625b058fbbb7712623 Mon Sep 17 00:00:00 2001 From: "Xingjun.Wang" Date: Wed, 13 May 2026 17:46:20 +0800 Subject: [PATCH 14/23] [Fix] Fix split detection (#1714) --- .../msdatasets/utils/hf_datasets_util.py | 125 ++++++++++++++++-- 1 file changed, 113 insertions(+), 12 deletions(-) diff --git a/modelscope/msdatasets/utils/hf_datasets_util.py b/modelscope/msdatasets/utils/hf_datasets_util.py index 40167606..8c89ecc9 100644 --- a/modelscope/msdatasets/utils/hf_datasets_util.py +++ b/modelscope/msdatasets/utils/hf_datasets_util.py @@ -473,6 +473,70 @@ def _hf_fs_open(self, path, mode='rb', **kwargs): return _hf_fs_open_original(self, path, mode=mode, **kwargs) +class _DryRunDownloadManager: + """Minimal download-manager stub for split discovery without I/O. + + Returns placeholder paths for all download/extract calls, allowing + _split_generators() to execute and return SplitGenerator objects + (which carry split names) without triggering actual network or disk I/O. + """ + + _PLACEHOLDER = os.devnull + + def download(self, url_or_urls): + return self._map(url_or_urls) + + def download_and_extract(self, url_or_urls): + return self._map(url_or_urls) + + def extract(self, path_or_paths): + return self._map(path_or_paths) + + def download_custom(self, url_or_urls, custom_download): + return self._map(url_or_urls) + + def iter_archive(self, path): + return iter([]) + + def iter_files(self, paths): + return iter([]) + + @property + def manual_dir(self): + return None + + @property + def is_streaming(self): + return False + + def _map(self, input_): + if isinstance(input_, dict): + return {k: self._PLACEHOLDER for k in input_} + if isinstance(input_, (list, tuple, set)): + return type(input_)(self._PLACEHOLDER for _ in input_) + return self._PLACEHOLDER + + +def _discover_splits_from_builder(builder_instance): + """Discover available split names by dry-running _split_generators(). + + For script-based datasets that lack README split metadata, this calls + the builder's _split_generators() with a stub download manager to + extract split names without performing any actual downloads. + + Returns: + A set of split name strings, or an empty set if discovery fails. + """ + if not hasattr(builder_instance, '_split_generators'): + return set() + try: + generators = builder_instance._split_generators(_DryRunDownloadManager()) + return {str(sg.name) for sg in generators} + except Exception as e: + logger.debug(f'Failed to discover splits from builder: {e}') + return set() + + def _validate_split_exists(builder_instance, split): """Fail-fast check: raise ValueError before downloading if the requested split does not exist in the dataset metadata. @@ -493,20 +557,29 @@ def _validate_split_exists(builder_instance, split): if not split_names: return - # Prefer info.splits (original metadata); fall back to data_files keys + # Source 1: info.splits (original metadata from README) available = set() info = getattr(builder_instance, 'info', None) if info is not None and info.splits: available = set(info.splits.keys()) + # Source 2: config.data_files keys if not available: config = getattr(builder_instance, 'config', None) data_files = getattr(config, 'data_files', None) if isinstance(data_files, dict): available = set(data_files.keys()) + # Source 3: dry-run _split_generators() for script-based datasets if not available: - return # Cannot determine available splits; let downstream handle + available = _discover_splits_from_builder(builder_instance) + + if not available: + logger.debug( + 'Cannot determine available splits from dataset metadata; ' + 'split validation skipped. Invalid splits will be caught downstream.' + ) + return missing = split_names - available if missing: @@ -642,30 +715,58 @@ class DatasetsWrapperHF: if dataset_info_only: ret_dict = {} + + # Case 1: Local .py script file if isinstance(path, str) and path.endswith('.py') and os.path.exists(path): from datasets import get_dataset_config_names subset_list = get_dataset_config_names(path) ret_dict = {_subset: [] for _subset in subset_list} return ret_dict - if builder_instance is None or not hasattr(builder_instance, - 'builder_configs'): - logger.error(f'No builder_configs found for {path} dataset.') + if builder_instance is None: + logger.error(f'No builder instance created for {path} dataset.') return ret_dict - _tmp_builder_configs = builder_instance.builder_configs - for tmp_config_name, tmp_builder_config in _tmp_builder_configs.items(): - tmp_config_name = str(tmp_config_name) - if hasattr(tmp_builder_config, 'data_files') and tmp_builder_config.data_files is not None: - ret_dict[tmp_config_name] = [str(item) for item in list(tmp_builder_config.data_files.keys())] + # Case 2: Try builder_configs with data_files (packaged datasets) + _tmp_builder_configs = getattr(builder_instance, 'builder_configs', None) + if _tmp_builder_configs: + if hasattr(_tmp_builder_configs, 'items'): + configs_iter = _tmp_builder_configs.items() else: - ret_dict[tmp_config_name] = [] + configs_iter = [(getattr(c, 'name', 'default'), c) for c in _tmp_builder_configs] + for tmp_config_name, tmp_builder_config in configs_iter: + tmp_config_name = str(tmp_config_name) + if hasattr(tmp_builder_config, 'data_files') and tmp_builder_config.data_files is not None: + ret_dict[tmp_config_name] = [str(item) for item in list(tmp_builder_config.data_files.keys())] + + # Case 3: Fallback for script datasets — use info.splits or dry-run discovery + if not ret_dict or all(not v for v in ret_dict.values()): + config_name = getattr(builder_instance, 'config_name', 'default') or 'default' + splits = [] + + # Try info.splits (from README metadata) + info = getattr(builder_instance, 'info', None) + if info is not None and info.splits: + splits = sorted(info.splits.keys()) + + # Fallback: dry-run _split_generators() + if not splits: + discovered = _discover_splits_from_builder(builder_instance) + if discovered: + splits = sorted(discovered) + + if splits: + ret_dict = {config_name: splits} + elif not ret_dict: + ret_dict = {config_name: []} + return ret_dict + _validate_split_exists(builder_instance, split) + if streaming: return builder_instance.as_streaming_dataset(split=split) - _validate_split_exists(builder_instance, split) _align_builder_splits_with_data_files(builder_instance, split) builder_instance.download_and_prepare( From 6f347a203cae63b421c4a08e88ae89ed6192a039 Mon Sep 17 00:00:00 2001 From: xvxuopop <127376094+xvxuopop@users.noreply.github.com> Date: Thu, 14 May 2026 10:36:31 +0800 Subject: [PATCH 15/23] update npu dockerfile (#1716) --- docker/Dockerfile.ascend | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.ascend b/docker/Dockerfile.ascend index c9f4e80e..442a08fe 100644 --- a/docker/Dockerfile.ascend +++ b/docker/Dockerfile.ascend @@ -28,8 +28,8 @@ RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \ # ---------- Install vllm + vllm-ascend ---------- RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ if [ -f /usr/local/Ascend/nnal/atb/set_env.sh ]; then source /usr/local/Ascend/nnal/atb/set_env.sh; fi && \ - git clone --depth 1 --branch v0.14.0 https://github.com/vllm-project/vllm && \ - git clone --depth 1 --branch v0.14.0rc1 https://github.com/vllm-project/vllm-ascend.git + git clone --depth 1 --branch v0.18.0 https://github.com/vllm-project/vllm && \ + git clone --depth 1 --branch v0.18.0 https://github.com/vllm-project/vllm-ascend.git RUN ARCH=$(uname -m) && \ source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ From cddbabaed5e729940f3462260a69205f8f7a44ff Mon Sep 17 00:00:00 2001 From: tastelikefeet <58414341+tastelikefeet@users.noreply.github.com> Date: Fri, 15 May 2026 14:06:55 +0800 Subject: [PATCH 16/23] Fix pipeline parameter (#1717) --- modelscope/pipelines/builder.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modelscope/pipelines/builder.py b/modelscope/pipelines/builder.py index b0095b73..5704a4d9 100644 --- a/modelscope/pipelines/builder.py +++ b/modelscope/pipelines/builder.py @@ -245,7 +245,10 @@ def pipeline(task: str = None, if preprocessor is not None: cfg.preprocessor = preprocessor - return build_pipeline(cfg, task_name=task) + return build_pipeline( + cfg, + task_name=task, + default_args={'trust_remote_code': trust_remote_code}) def add_default_pipeline_info(task: str, From 99602db2736f3b0c9946fcb7825be27df78b6271 Mon Sep 17 00:00:00 2001 From: Jintao Huang Date: Mon, 18 May 2026 16:06:45 +0800 Subject: [PATCH 17/23] fix --- docker/Dockerfile.ubuntu.old | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/Dockerfile.ubuntu.old b/docker/Dockerfile.ubuntu.old index 6784702e..f8f7f2fc 100644 --- a/docker/Dockerfile.ubuntu.old +++ b/docker/Dockerfile.ubuntu.old @@ -1,5 +1,8 @@ FROM {base_image} +ARG CUR_TIME={cur_time} +RUN echo $CUR_TIME + RUN cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b {modelscope_branch} --single-branch https://github.com/modelscope/modelscope.git && \ cd modelscope && pip install . -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ cd / && rm -fr /tmp/modelscope && pip cache purge; From 6bcb1f195b377f9e15bb0a604111e5e116a73ed1 Mon Sep 17 00:00:00 2001 From: Jintao Huang Date: Mon, 18 May 2026 16:09:55 +0800 Subject: [PATCH 18/23] fix --- docker/build_image.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/build_image.py b/docker/build_image.py index daef43e2..de215aa1 100644 --- a/docker/build_image.py +++ b/docker/build_image.py @@ -141,6 +141,7 @@ class OldCPUImageBuilder(Builder): content = content.replace('{base_image}', old_cpu_image) content = content.replace('{modelscope_branch}', self.args.modelscope_branch) + content = content.replace('{cur_time}', formatted_time) return content def image(self) -> str: @@ -198,6 +199,7 @@ class OldGPUImageBuilder(Builder): content = content.replace('{base_image}', old_gpu_image) content = content.replace('{modelscope_branch}', self.args.modelscope_branch) + content = content.replace('{cur_time}', formatted_time) return content def image(self) -> str: From 46d653b079256ae4938d549a10cfed5e7a63f55c Mon Sep 17 00:00:00 2001 From: tastelikefeet <58414341+tastelikefeet@users.noreply.github.com> Date: Tue, 19 May 2026 12:53:02 +0800 Subject: [PATCH 19/23] Add whitelist to iic and damo (#1720) --- modelscope/models/base/base_model.py | 5 +++-- modelscope/pipelines/builder.py | 5 ++++- modelscope/preprocessors/base.py | 1 + modelscope/trainers/builder.py | 7 +++++-- modelscope/utils/plugins.py | 11 +++++++++++ 5 files changed, 24 insertions(+), 5 deletions(-) diff --git a/modelscope/models/base/base_model.py b/modelscope/models/base/base_model.py index 0a693de0..3d735630 100644 --- a/modelscope/models/base/base_model.py +++ b/modelscope/models/base/base_model.py @@ -15,7 +15,7 @@ from modelscope.utils.constant import DEFAULT_MODEL_REVISION, Invoke, ModelFile from modelscope.utils.device import verify_device from modelscope.utils.logger import get_logger from modelscope.utils.plugins import (filter_plugin_in_whitelist, - register_modelhub_repo, + is_trusted_group, register_modelhub_repo, register_plugins_repo) logger = get_logger() @@ -137,7 +137,8 @@ class Model(ABC): kwargs.pop(Invoke.KEY) else: invoked_by = Invoke.PRETRAINED - + trust_remote_code = trust_remote_code or is_trusted_group( + model_name_or_path) ignore_file_pattern = kwargs.pop('ignore_file_pattern', None) if osp.exists(model_name_or_path): local_model_dir = model_name_or_path diff --git a/modelscope/pipelines/builder.py b/modelscope/pipelines/builder.py index 5704a4d9..37eb29d7 100644 --- a/modelscope/pipelines/builder.py +++ b/modelscope/pipelines/builder.py @@ -13,7 +13,7 @@ from modelscope.utils.hub import read_config from modelscope.utils.import_utils import is_transformers_available from modelscope.utils.logger import get_logger from modelscope.utils.plugins import (filter_plugin_in_whitelist, - register_modelhub_repo, + is_trusted_group, register_modelhub_repo, register_plugins_repo) from modelscope.utils.registry import Registry, build_from_cfg from modelscope.utils.task_utils import is_embedding_task @@ -117,6 +117,9 @@ def pipeline(task: str = None, if task is None and pipeline_name is None: raise ValueError('task or pipeline_name is required') + model_id = model[0] if isinstance(model, + list) and len(model) > 0 else model + trust_remote_code = trust_remote_code or is_trusted_group(model_id) pipeline_props = None if pipeline_name is None: # get default pipeline for this task diff --git a/modelscope/preprocessors/base.py b/modelscope/preprocessors/base.py index d3138a3e..f87cebe7 100644 --- a/modelscope/preprocessors/base.py +++ b/modelscope/preprocessors/base.py @@ -208,6 +208,7 @@ class Preprocessor(ABC): revision: Optional[str] = DEFAULT_MODEL_REVISION, cfg_dict: Config = None, preprocessor_mode=ModeKeys.INFERENCE, + trust_remote_code=False, **kwargs): """Instantiate a preprocessor from local directory or remote model repo. Note that when loading from remote, the model revision can be specified. diff --git a/modelscope/trainers/builder.py b/modelscope/trainers/builder.py index 2a04343c..723b6247 100644 --- a/modelscope/trainers/builder.py +++ b/modelscope/trainers/builder.py @@ -5,7 +5,7 @@ from modelscope.pipelines.util import is_official_hub_path from modelscope.utils.constant import DEFAULT_MODEL_REVISION from modelscope.utils.hub import read_config from modelscope.utils.plugins import (filter_plugin_in_whitelist, - register_modelhub_repo, + is_trusted_group, register_modelhub_repo, register_plugins_repo) from modelscope.utils.registry import Registry, build_from_cfg @@ -27,7 +27,10 @@ def build_trainer(name: str = Trainers.default, default_args: dict = None): default_args = default_args or {} model = default_args.get('model', None) model_revision = default_args.get('model_revision', DEFAULT_MODEL_REVISION) - trust_remote_code = default_args.get('trust_remote_code', False) + model_id = model[0] if isinstance(model, + list) and len(model) > 0 else model + trust_remote_code = default_args.get('trust_remote_code', + False) or is_trusted_group(model_id) if isinstance(model, str) \ or (isinstance(model, list) and isinstance(model[0], str)): diff --git a/modelscope/utils/plugins.py b/modelscope/utils/plugins.py index fa4cb877..9ebb1215 100644 --- a/modelscope/utils/plugins.py +++ b/modelscope/utils/plugins.py @@ -61,6 +61,17 @@ def filter_plugin_in_whitelist(plugins): ] +def is_trusted_group(model_id: Optional[str] = None) -> bool: + if not isinstance(model_id, str) or os.path.exists(model_id): + return False + pattern = re.compile(r'^([^/]+)/[^/]+$') + m = pattern.match(model_id) + if not m: + return False + group = m.group(1) + return group in ['iic', 'damo', 'modelscope'] + + @contextmanager def pushd(new_dir: str, verbose: bool = False): """ From 4c750ee490df95a1cf8f5276b117899e7d654360 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=A8=E6=B3=93?= Date: Tue, 19 May 2026 15:05:21 +0800 Subject: [PATCH 20/23] fix pipelines in plugin --- modelscope/utils/registry.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modelscope/utils/registry.py b/modelscope/utils/registry.py index ae81cfc4..a323d2a8 100644 --- a/modelscope/utils/registry.py +++ b/modelscope/utils/registry.py @@ -205,6 +205,8 @@ def build_from_cfg(cfg, raise TypeError( f'type must be a str or valid type, but got {type(obj_type)}') try: + if not obj_cls.__module__.startswith('modelscope'): + args.pop('trust_remote_code', None) if hasattr(obj_cls, '_instantiate'): return obj_cls._instantiate(**args) else: From 0df6b5c5a5bd46c8b918e0f3a15d0e1f842f2f22 Mon Sep 17 00:00:00 2001 From: tastelikefeet <58414341+tastelikefeet@users.noreply.github.com> Date: Wed, 20 May 2026 23:12:03 +0800 Subject: [PATCH 21/23] Fix adaseq trust_remote_code (#1721) --- modelscope/models/base/base_model.py | 12 +++++++----- modelscope/models/cv/anydoor/anydoor_model.py | 2 +- .../image_view_transform_infer.py | 2 +- .../models/cv/tinynas_detection/detector.py | 2 +- .../cv/video_depth_estimation/dro_model.py | 2 +- .../pipelines/audio/linear_aec_pipeline.py | 3 ++- modelscope/pipelines/builder.py | 17 +++++++++++------ .../disco_guided_diffusion.py | 9 ++------- modelscope/trainers/builder.py | 7 ++++--- modelscope/utils/automodel_utils.py | 2 +- modelscope/utils/plugins.py | 11 ----------- 11 files changed, 31 insertions(+), 38 deletions(-) diff --git a/modelscope/models/base/base_model.py b/modelscope/models/base/base_model.py index 3d735630..4a1b6494 100644 --- a/modelscope/models/base/base_model.py +++ b/modelscope/models/base/base_model.py @@ -15,7 +15,7 @@ from modelscope.utils.constant import DEFAULT_MODEL_REVISION, Invoke, ModelFile from modelscope.utils.device import verify_device from modelscope.utils.logger import get_logger from modelscope.utils.plugins import (filter_plugin_in_whitelist, - is_trusted_group, register_modelhub_repo, + register_modelhub_repo, register_plugins_repo) logger = get_logger() @@ -32,7 +32,9 @@ class Model(ABC): device_name = kwargs.get('device', 'gpu') verify_device(device_name) self._device_name = device_name - self.trust_remote_code = kwargs.get('trust_remote_code', False) + self.trust_remote_code = kwargs.get( + 'trust_remote_code', + False) or check_model_from_owner_group(model_dir) def __call__(self, *args, **kwargs) -> Dict[str, Any]: return self.postprocess(self.forward(*args, **kwargs)) @@ -137,8 +139,8 @@ class Model(ABC): kwargs.pop(Invoke.KEY) else: invoked_by = Invoke.PRETRAINED - trust_remote_code = trust_remote_code or is_trusted_group( - model_name_or_path) + _model_trusted = check_model_from_owner_group(model_name_or_path) + trust_remote_code = trust_remote_code or _model_trusted ignore_file_pattern = kwargs.pop('ignore_file_pattern', None) if osp.exists(model_name_or_path): local_model_dir = model_name_or_path @@ -205,7 +207,7 @@ class Model(ABC): 'Please make sure that you can trust the external codes.') register_modelhub_repo(local_model_dir, allow_remote=trust_remote_code) default_args = {} - if trust_remote_code: + if trust_remote_code and not _model_trusted: default_args = {'trust_remote_code': trust_remote_code} register_plugins_repo(plugins) for k, v in kwargs.items(): diff --git a/modelscope/models/cv/anydoor/anydoor_model.py b/modelscope/models/cv/anydoor/anydoor_model.py index ff9216c1..ef618f0d 100644 --- a/modelscope/models/cv/anydoor/anydoor_model.py +++ b/modelscope/models/cv/anydoor/anydoor_model.py @@ -343,7 +343,7 @@ class ControlLDM(LatentDiffusion, Model): self.only_mid_control = only_mid_control self.control_scales = [1.0] * 13 self.trust_remote_code = kwargs.get('trust_remote_code', False) - self.check_trust_remote_code() + self.check_trust_remote_code(self.model_dir) @torch.no_grad() def get_input(self, batch, k, bs=None, *args, **kwargs): diff --git a/modelscope/models/cv/image_view_transform/image_view_transform_infer.py b/modelscope/models/cv/image_view_transform/image_view_transform_infer.py index e7cac98e..0281f5fa 100644 --- a/modelscope/models/cv/image_view_transform/image_view_transform_infer.py +++ b/modelscope/models/cv/image_view_transform/image_view_transform_infer.py @@ -68,7 +68,7 @@ class ImageViewTransform(TorchModel): self.model = None self.model = load_model_from_config( self.model, config, ckpt, device=self.device) - self.check_trust_remote_code() + self.check_trust_remote_code(model_dir=model_dir) def forward(self, model_path, x, y): pred_results = _infer(self.model, model_path, x, y, self.device) diff --git a/modelscope/models/cv/tinynas_detection/detector.py b/modelscope/models/cv/tinynas_detection/detector.py index dfab0889..e6e94f6b 100644 --- a/modelscope/models/cv/tinynas_detection/detector.py +++ b/modelscope/models/cv/tinynas_detection/detector.py @@ -29,7 +29,7 @@ class SingleStageDetector(TorchModel): init model by cfg """ super().__init__(model_dir, *args, **kwargs) - self.check_trust_remote_code() + self.check_trust_remote_code(model_dir=model_dir) config_path = osp.join(model_dir, self.config_name) config = parse_config(config_path) self.cfg = config diff --git a/modelscope/models/cv/video_depth_estimation/dro_model.py b/modelscope/models/cv/video_depth_estimation/dro_model.py index f39ee423..b32076f3 100644 --- a/modelscope/models/cv/video_depth_estimation/dro_model.py +++ b/modelscope/models/cv/video_depth_estimation/dro_model.py @@ -33,7 +33,7 @@ class DROEstimation(TorchModel): def __init__(self, model_dir: str, **kwargs): """str -- model file root.""" super().__init__(model_dir, **kwargs) - self.check_trust_remote_code() + self.check_trust_remote_code(model_dir=model_dir) model_path = osp.join(model_dir, ModelFile.TORCH_MODEL_FILE) diff --git a/modelscope/pipelines/audio/linear_aec_pipeline.py b/modelscope/pipelines/audio/linear_aec_pipeline.py index af0c2156..fb399476 100644 --- a/modelscope/pipelines/audio/linear_aec_pipeline.py +++ b/modelscope/pipelines/audio/linear_aec_pipeline.py @@ -70,7 +70,8 @@ class LinearAECPipeline(Pipeline): self.check_trust_remote_code( 'This pipeline requires `trust_remote_code=True` to load the module defined' ' in the `dey_mini.yaml`, setting this to True means you trust the code and files' - ' listed in this model repo.') + ' listed in this model repo.', + model_dir=model) self.use_cuda = torch.cuda.is_available() with open( diff --git a/modelscope/pipelines/builder.py b/modelscope/pipelines/builder.py index 37eb29d7..d379e3db 100644 --- a/modelscope/pipelines/builder.py +++ b/modelscope/pipelines/builder.py @@ -6,6 +6,7 @@ from typing import Any, Dict, List, Optional, Union from modelscope.hub.snapshot_download import snapshot_download from modelscope.metainfo import DEFAULT_MODEL_FOR_PIPELINE from modelscope.models.base import Model +from modelscope.utils.automodel_utils import check_model_from_owner_group from modelscope.utils.config import ConfigDict, check_config from modelscope.utils.constant import (DEFAULT_MODEL_REVISION, Invoke, Tasks, ThirdParty) @@ -13,7 +14,7 @@ from modelscope.utils.hub import read_config from modelscope.utils.import_utils import is_transformers_available from modelscope.utils.logger import get_logger from modelscope.utils.plugins import (filter_plugin_in_whitelist, - is_trusted_group, register_modelhub_repo, + register_modelhub_repo, register_plugins_repo) from modelscope.utils.registry import Registry, build_from_cfg from modelscope.utils.task_utils import is_embedding_task @@ -119,7 +120,8 @@ def pipeline(task: str = None, model_id = model[0] if isinstance(model, list) and len(model) > 0 else model - trust_remote_code = trust_remote_code or is_trusted_group(model_id) + _model_trusted = check_model_from_owner_group(model_id) + trust_remote_code = trust_remote_code or _model_trusted pipeline_props = None if pipeline_name is None: # get default pipeline for this task @@ -248,10 +250,13 @@ def pipeline(task: str = None, if preprocessor is not None: cfg.preprocessor = preprocessor - return build_pipeline( - cfg, - task_name=task, - default_args={'trust_remote_code': trust_remote_code}) + if _model_trusted: + return build_pipeline(cfg, task_name=task) + else: + return build_pipeline( + cfg, + task_name=task, + default_args={'trust_remote_code': trust_remote_code}) def add_default_pipeline_info(task: str, diff --git a/modelscope/pipelines/multi_modal/disco_guided_diffusion_pipeline/disco_guided_diffusion.py b/modelscope/pipelines/multi_modal/disco_guided_diffusion_pipeline/disco_guided_diffusion.py index 670efae2..ebe4f0c2 100644 --- a/modelscope/pipelines/multi_modal/disco_guided_diffusion_pipeline/disco_guided_diffusion.py +++ b/modelscope/pipelines/multi_modal/disco_guided_diffusion_pipeline/disco_guided_diffusion.py @@ -193,7 +193,8 @@ class DiscoDiffusionPipeline(DiffusersPipeline): self.check_trust_remote_code( 'This pipeline requires `trust_remote_code=True` to load the module defined' ' in `model_index.json`, setting this to True means you trust the code and files' - ' listed in this model repo.') + ' listed in this model repo.', + model_dir=model) model_path = model @@ -209,12 +210,6 @@ class DiscoDiffusionPipeline(DiffusersPipeline): if model_config['use_fp16']: self.unet.convert_to_fp16() - self.trust_remote_code = kwargs.get('trust_remote_code', False) - self.check_trust_remote_code( - 'This pipeline requires import modules listed in `model_index.json`, ' - 'please add `trust_remote_code=True` if you trust this model repo.' - ) - with open( os.path.join(model_path, 'model_index.json'), 'r', diff --git a/modelscope/trainers/builder.py b/modelscope/trainers/builder.py index 723b6247..fdef22f4 100644 --- a/modelscope/trainers/builder.py +++ b/modelscope/trainers/builder.py @@ -2,10 +2,11 @@ from modelscope.metainfo import Trainers from modelscope.pipelines.builder import normalize_model_input from modelscope.pipelines.util import is_official_hub_path +from modelscope.utils.automodel_utils import check_model_from_owner_group from modelscope.utils.constant import DEFAULT_MODEL_REVISION from modelscope.utils.hub import read_config from modelscope.utils.plugins import (filter_plugin_in_whitelist, - is_trusted_group, register_modelhub_repo, + register_modelhub_repo, register_plugins_repo) from modelscope.utils.registry import Registry, build_from_cfg @@ -29,8 +30,8 @@ def build_trainer(name: str = Trainers.default, default_args: dict = None): model_revision = default_args.get('model_revision', DEFAULT_MODEL_REVISION) model_id = model[0] if isinstance(model, list) and len(model) > 0 else model - trust_remote_code = default_args.get('trust_remote_code', - False) or is_trusted_group(model_id) + trust_remote_code = default_args.get( + 'trust_remote_code', False) or check_model_from_owner_group(model_id) if isinstance(model, str) \ or (isinstance(model, list) and isinstance(model[0], str)): diff --git a/modelscope/utils/automodel_utils.py b/modelscope/utils/automodel_utils.py index bf68da74..ba087b22 100644 --- a/modelscope/utils/automodel_utils.py +++ b/modelscope/utils/automodel_utils.py @@ -137,7 +137,7 @@ def check_model_from_owner_group(model_dir: str, Returns: bool: Whether the group can be trusted """ - if not model_dir: + if not model_dir or not isinstance(model_dir, str): return False if owner_group is None: owner_group = ['iic', 'damo'] diff --git a/modelscope/utils/plugins.py b/modelscope/utils/plugins.py index 9ebb1215..fa4cb877 100644 --- a/modelscope/utils/plugins.py +++ b/modelscope/utils/plugins.py @@ -61,17 +61,6 @@ def filter_plugin_in_whitelist(plugins): ] -def is_trusted_group(model_id: Optional[str] = None) -> bool: - if not isinstance(model_id, str) or os.path.exists(model_id): - return False - pattern = re.compile(r'^([^/]+)/[^/]+$') - m = pattern.match(model_id) - if not m: - return False - group = m.group(1) - return group in ['iic', 'damo', 'modelscope'] - - @contextmanager def pushd(new_dir: str, verbose: bool = False): """ From 7d89214eb12efc94f1b440e6199fa25cc9fc60b9 Mon Sep 17 00:00:00 2001 From: Jintao Huang Date: Thu, 21 May 2026 00:40:50 +0800 Subject: [PATCH 22/23] bump version --- modelscope/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modelscope/version.py b/modelscope/version.py index 76be6b7e..1105d343 100644 --- a/modelscope/version.py +++ b/modelscope/version.py @@ -1,5 +1,5 @@ # Make sure to modify __release_datetime__ to release time when making official release. -__version__ = '1.37.0' +__version__ = '1.37.1' # default release datetime for branches under active development is set # to be a time far-far-away-into-the-future -__release_datetime__ = '2026-05-09 23:59:59' +__release_datetime__ = '2026-05-20 23:59:59' From 121f4150fa9615882414bf424199e15be09c8ae5 Mon Sep 17 00:00:00 2001 From: Jintao Huang Date: Thu, 21 May 2026 16:27:12 +0800 Subject: [PATCH 23/23] fix cpu image --- docker/Dockerfile.ubuntu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu index d2d356a3..adb4abe3 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.ubuntu @@ -74,7 +74,7 @@ RUN if [ "$IMAGE_TYPE" = "gpu" ]; then \ cd / && rm -fr /tmp/apex && pip cache purge; \ pip install --no-cache-dir "megatron-core==0.16.*" -U; \ elif [ "$IMAGE_TYPE" = "cpu" ]; then \ - pip install --no-cache-dir huggingface-hub transformers peft diffusers -U; \ + pip install --no-cache-dir huggingface-hub "transformers<5.9" peft diffusers -U; \ else \ pip install "transformers<5.0" "tokenizers<0.22" "trl<0.23" "diffusers<0.35" --no-dependencies; \ fi