Merge remote-tracking branch 'origin/release/1.33' into build_swift_image

This commit is contained in:
Jintao Huang
2026-01-08 10:34:24 +08:00
12 changed files with 306 additions and 18 deletions

120
docker/Dockerfile.ascend Normal file
View File

@@ -0,0 +1,120 @@
FROM {base_image}
RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \
pip config set install.trusted-host mirrors.aliyun.com
# Prepare required system dependencies
RUN apt-get update -y && \
apt-get install -y --no-install-recommends gcc g++ cmake libnuma-dev wget git curl jq vim build-essential && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
pip install --upgrade pip setuptools packaging && \
pip cache purge
{extra_content}
# Prepare repositories with low update frequency
RUN ARCH=$(uname -m) && \
# Set extra pip index for x86_64 platform
echo "[LOG INFO] Detected architecture: $ARCH" && \
if [ "$ARCH" = "x86_64" ]; then \
pip config set global.extra-index-url "https://download.pytorch.org/whl/cpu/"; \
fi && \
# Clone libs
git clone --depth 1 --branch v0.11.0 https://github.com/vllm-project/vllm && \
git clone --depth 1 --branch v0.11.0rc1 https://github.com/vllm-project/vllm-ascend.git && \
git clone https://gitcode.com/Ascend/MindSpeed.git && \
cd MindSpeed && git checkout f2b0977e && cd .. && \
git clone --depth 1 --branch core_v0.12.1 https://github.com/NVIDIA/Megatron-LM.git
# Install repositories with low update frequency
RUN ARCH=$(uname -m) && \
# Export and source env
if [ "$ARCH" = "aarch64" ]; then \
export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/8.3.RC1/aarch64-linux/devlib/linux/aarch64:$LD_LIBRARY_PATH; \
elif [ "$ARCH" = "x86_64" ]; then \
export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/8.3.RC1/x86_64-linux/devlib/linux/x86_64/:$LD_LIBRARY_PATH; \
fi && \
source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
source /usr/local/Ascend/nnal/atb/set_env.sh && \
# Install torch & torch_npu & torchvision
pip install torch==2.7.1 torch_npu==2.7.1 torchvision==0.22.1 && \
# Install vllm
cd vllm && VLLM_TARGET_DEVICE=empty pip install -v -e . && cd .. && \
# Install vllm-ascend
cd vllm-ascend && pip install -v -e . && cd .. && \
# Install MindSpeed & Megatron
pip install -e MindSpeed && \
# Clear extra files
rm -rf /tmp/* /var/tmp/* && \
pip cache purge
ENV PYTHONPATH ${PYTHONPATH}:/Megatron-LM
ARG INSTALL_MS_DEPS={install_ms_deps}
# install dependencies
COPY requirements /var/modelscope
RUN pip uninstall ms-swift modelscope -y && pip --no-cache-dir install pip==23.* -U && \
if [ "$INSTALL_MS_DEPS" = "True" ]; then \
pip --no-cache-dir install omegaconf==2.0.6 && \
pip install 'editdistance==0.8.1' && \
pip install --no-cache-dir 'cython<=0.29.36' versioneer 'numpy<2.0' -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
pip install --no-cache-dir -r /var/modelscope/framework.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
pip install --no-cache-dir -r /var/modelscope/audio.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
pip install --no-cache-dir -r /var/modelscope/cv.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
pip install --no-cache-dir -r /var/modelscope/multi-modal.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
pip install --no-cache-dir -r /var/modelscope/nlp.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
pip install --no-cache-dir -r /var/modelscope/science.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
pip install --no-cache-dir -r /var/modelscope/tests.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
pip install --no-cache-dir -r /var/modelscope/server.txt && \
pip install --no-cache-dir https://modelscope.oss-cn-beijing.aliyuncs.com/packages/imageio_ffmpeg-0.4.9-py3-none-any.whl --no-dependencies --force && \
pip install adaseq pai-easycv && \
pip install --no-cache-dir 'scipy<1.13.0' && \
pip install --no-cache-dir funtextprocessing typeguard==2.13.3 scikit-learn -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
pip install --no-cache-dir text2sql_lgesql==1.3.0 git+https://github.com/jin-s13/xtcocoapi.git@v1.14 git+https://github.com/gatagat/lap.git@v0.4.0 -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --force --no-deps && \
pip install --no-cache-dir mmcls>=0.21.0 mmdet>=2.25.0 decord>=0.6.0 mpi4py paint_ldm ipykernel fasttext -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
pip uninstall ddpm_guided_diffusion -y && \
pip install --no-cache-dir 'blobfile>=1.0.5' && \
pip install 'ddpm_guided_diffusion' -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --no-index && \
pip uninstall shotdetect_scenedetect_lgss -y && \
pip install 'shotdetect_scenedetect_lgss' -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --no-index && \
pip uninstall MinDAEC -y && \
pip install https://modelscope.oss-cn-beijing.aliyuncs.com/releases/dependencies/MinDAEC-0.0.2-py3-none-any.whl && \
pip cache purge; \
else \
pip install --no-cache-dir -r /var/modelscope/framework.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
pip cache purge; \
fi
ARG CUR_TIME={cur_time}
RUN echo $CUR_TIME
RUN curl -fsSL https://ollama.com/install.sh | sh && \
pip install --no-cache-dir -U funasr scikit-learn && \
pip install --no-cache-dir -U qwen_vl_utils qwen_omni_utils librosa timm transformers accelerate peft trl safetensors && \
cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b {swift_branch} --single-branch https://github.com/modelscope/ms-swift.git && \
cd ms-swift && pip install .[llm] && \
pip install .[eval] && pip install evalscope -U --no-dependencies && pip install ms-agent -U --no-dependencies && \
cd / && rm -fr /tmp/ms-swift && pip cache purge; \
cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b {modelscope_branch} --single-branch https://github.com/modelscope/modelscope.git && \
cd modelscope && pip install . -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \
cd / && rm -fr /tmp/modelscope && pip cache purge; \
pip install --no-cache-dir transformers diffusers timm>=0.9.0 && pip cache purge; \
pip install --no-cache-dir omegaconf==2.3.0 && pip cache purge;
RUN if [ "$INSTALL_MS_DEPS" = "True" ]; then \
pip install --no-cache-dir huggingface-hub transformers peft -U; \
fi;
ENV SETUPTOOLS_USE_DISTUTILS=stdlib
ENV VLLM_USE_MODELSCOPE=True
ENV LMDEPLOY_USE_MODELSCOPE=True
ENV MODELSCOPE_CACHE=/mnt/workspace/.cache/modelscope/hub
# Show install results
RUN pip list
SHELL ["/bin/bash", "-c"]

View File

@@ -417,6 +417,38 @@ RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy
return os.system(f'docker push {image_tag2}')
class AscendSwiftImageBuilder(SwiftImageBuilder):
def init_args(self, args) -> Any:
if not args.base_image:
# other vision search for: https://hub.docker.com/r/ascendai/cann/tags
args.base_image = 'swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:8.3.rc1-a3-ubuntu22.04-py3.11'
return super().init_args(args)
def generate_dockerfile(self) -> str:
extra_content = """
RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy
"""
with open('docker/Dockerfile.ascend', 'r') as f:
content = f.read()
content = content.replace('{base_image}', self.args.base_image)
content = content.replace('{extra_content}', extra_content)
content = content.replace('{cur_time}', formatted_time)
content = content.replace('{install_ms_deps}', 'False')
content = content.replace('{modelscope_branch}',
self.args.modelscope_branch)
content = content.replace('{swift_branch}', self.args.swift_branch)
return content
def image(self) -> str:
return (
f'{docker_registry}:{self.args.base_image.split(":")[-1]}-torch2.7.1'
f'-{self.args.modelscope_version}-ascend-swift-test')
def push(self):
return 0
parser = argparse.ArgumentParser()
parser.add_argument('--base_image', type=str, default=None)
parser.add_argument('--image_type', type=str)
@@ -450,6 +482,8 @@ elif args.image_type.lower() == 'llm':
builder_cls = LLMImageBuilder
elif args.image_type.lower() == 'swift':
builder_cls = SwiftImageBuilder
elif args.image_type.lower() == 'ascend_swift':
builder_cls = AscendSwiftImageBuilder
else:
raise ValueError(f'Unsupported image_type: {args.image_type}')

View File

@@ -131,7 +131,7 @@ else:
if module is not None:
module = _patch_pretrained_class([module], wrap=True)
else:
raise ImportError(
raise AttributeError(
f'Cannot import available module of {name} in modelscope,'
f' or related packages({hf_pkgs})')
return module[0]

View File

@@ -129,6 +129,11 @@ class CreateCMD(CLICommand):
help=
'Source of the AIGC model. `USER_UPLOAD`, `TRAINED_FROM_MODELSCOPE` or `TRAINED_FROM_ALIYUN_FC`.'
)
aigc_group.add_argument(
'--base_model_sub_type',
type=str,
default='',
help='Base model sub type, e.g., Qwen_Edit_2509')
parser.set_defaults(func=subparser_func)
@@ -191,6 +196,7 @@ class CreateCMD(CLICommand):
base_model_id=self.args.base_model_id,
path_in_repo=self.args.path_in_repo,
model_source=self.args.model_source,
base_model_sub_type=self.args.base_model_sub_type,
)
# Convert visibility string to int for the API call

View File

@@ -19,7 +19,8 @@ from http import HTTPStatus
from http.cookiejar import CookieJar
from os.path import expanduser
from pathlib import Path
from typing import Any, BinaryIO, Dict, Iterable, List, Optional, Tuple, Union
from typing import (Any, BinaryIO, Dict, Iterable, List, Literal, Optional,
Tuple, Union)
from urllib.parse import urlencode
import json
@@ -304,6 +305,7 @@ class HubApi:
'ModelPath': aigc_model.model_path,
'TriggerWords': aigc_model.trigger_words,
'ModelSource': aigc_model.model_source,
'SubVisionFoundation': aigc_model.base_model_sub_type,
})
if aigc_model.official_tags:
@@ -394,7 +396,9 @@ class HubApi:
'WeightsName': aigc_model.weight_filename,
'WeightsSha256': aigc_model.weight_sha256,
'WeightsSize': aigc_model.weight_size,
'TriggerWords': aigc_model.trigger_words
'TriggerWords': aigc_model.trigger_words,
'AigcType': aigc_model.aigc_type,
'VisionFoundation': aigc_model.base_model_type
}
else:
@@ -455,7 +459,7 @@ class HubApi:
model_id: str,
revision: Optional[str] = DEFAULT_MODEL_REVISION,
endpoint: Optional[str] = None
) -> str:
) -> dict:
"""Get model information at ModelScope
Args:
@@ -2304,13 +2308,21 @@ class HubApi:
allow_patterns = allow_patterns if allow_patterns else None
ignore_patterns = ignore_patterns if ignore_patterns else None
# Ignore .git folder
# Ignore .git .cache folders
if ignore_patterns is None:
ignore_patterns = []
elif isinstance(ignore_patterns, str):
ignore_patterns = [ignore_patterns]
ignore_patterns += DEFAULT_IGNORE_PATTERNS
# Cover the ignore patterns if both allow and ignore patterns are provided
if allow_patterns is not None:
if '**' in allow_patterns:
ignore_patterns = []
ignore_patterns = [
p for p in ignore_patterns if p not in allow_patterns
]
commit_message = (
commit_message if commit_message is not None else f'Upload to {repo_id} on ModelScope hub'
)
@@ -2867,6 +2879,91 @@ class HubApi:
'total_files': len(to_delete)
}
def set_repo_visibility(self,
repo_id: str,
repo_type: Literal['model', 'dataset'],
visibility: Literal['private', 'public'],
token: Union[str, None] = None
) -> dict:
"""
Set the visibility of a repo.
Args:
repo_id (str): The repo id in the format of `owner_name/repo_name`.
repo_type (Literal['model', 'dataset']): The repo type, `model` or `dataset`.
visibility (Literal['private', 'public']): The visibility to set, `private` or `public`.
token (Union[str, None]): The access token. If None, will use the cookies from the local cache.
See `https://modelscope.cn/my/myaccesstoken` to get your token.
Returns:
dict: The response from the server.
"""
if not repo_id:
raise ValueError('The arg `repo_id` cannot be empty!')
if visibility not in ['private', 'public']:
raise ValueError(f'Invalid visibility: {visibility}, supported visibilities: `private`, `public`')
visibility_map: Dict[str, int] = {v: k for k, v in VisibilityMap.items()}
visibility_code: int = visibility_map.get(visibility, 5)
cookies = self.get_cookies(access_token=token, cookies_required=True)
if repo_type == REPO_TYPE_MODEL:
model_info = self.get_model(model_id=repo_id)
path = f'{self.endpoint}/api/v1/models/{repo_id}'
tasks = model_info.get('Tasks')
model_tasks = ''
if isinstance(tasks, list) and tasks:
first = tasks[0]
if isinstance(first, dict) and first:
model_tasks = first.get('name')
payload = {
'ChineseName': model_info.get('ChineseName', ''),
'ModelFramework': model_info.get('ModelFramework', 'Pytorch'),
'Visibility': visibility_code,
'ProtectedMode': 2,
'ApprovalMode': model_info.get('ApprovalMode', 2),
'Description': model_info.get('Description', ''),
'AigcType': model_info.get('AigcType', ''),
'VisionFoundation': model_info.get('VisionFoundation', ''),
'ModelCover': model_info.get('ModelCover', ''),
'SubScientificField': model_info.get('SubScientificField', None),
'ScientificField': model_info.get('NEXA', {}).get('ScientificField', ''),
'Source': model_info.get('NEXA', {}).get('Source', ''),
'ModelTask': model_tasks,
'License': model_info.get('License', ''),
}
elif repo_type == REPO_TYPE_DATASET:
repo_id_parts = repo_id.split('/')
if len(repo_id_parts) != 2 or not all(repo_id_parts):
raise ValueError(f'Invalid dataset repo_id: {repo_id}, should be in format of `owner/dataset_name`')
dataset_idx, _ = self.get_dataset_id_and_type(
dataset_name=repo_id_parts[1],
namespace=repo_id_parts[0],
)
path = f'{self.endpoint}/api/v1/datasets/{dataset_idx}'
payload = {
'Visibility': visibility_code,
'ProtectedMode': 2,
}
else:
raise ValueError(f'Invalid repo type: {repo_type}, supported repos: {REPO_TYPE_SUPPORT}')
r = self.session.put(
path,
json=payload,
cookies=cookies,
headers=self.builder_headers(self.headers))
raise_for_http_status(r)
resp = r.json()
raise_on_error(resp)
return resp
class ModelScopeConfig:
path_credential = expanduser(MODELSCOPE_CREDENTIALS_PATH)

View File

@@ -120,9 +120,9 @@ def handle_http_response(response: requests.Response,
http_error_msg = 'The request model: %s does not exist!' % (model_id)
elif 403 == response.status_code:
if cookies is None:
http_error_msg = f'Authentication token does not exist, \
failed to access model {model_id} which may not exist \
or may be private. Please login first.'
http_error_msg = (
f'Authentication token does not exist, failed to access model {model_id} '
'which may not exist or may be private. Please login first.')
else:
http_error_msg = f'The authentication token is invalid, failed to access model {model_id}.'

View File

@@ -19,6 +19,7 @@ from modelscope.utils.logger import get_logger
from modelscope.utils.thread_utils import thread_executor
from .api import HubApi, ModelScopeConfig
from .callback import ProgressCallback
from .constants import DEFAULT_MAX_WORKERS
from .errors import InvalidParameter
from .file_download import (create_temporary_directory_and_cache,
download_file, get_file_download_url)
@@ -42,7 +43,7 @@ def snapshot_download(
local_dir: Optional[str] = None,
allow_patterns: Optional[Union[List[str], str]] = None,
ignore_patterns: Optional[Union[List[str], str]] = None,
max_workers: int = 8,
max_workers: Optional[int] = None,
repo_id: str = None,
repo_type: Optional[str] = REPO_TYPE_MODEL,
enable_file_lock: Optional[bool] = None,
@@ -112,6 +113,8 @@ def snapshot_download(
f'Invalid repo type: {repo_type}, only support: {REPO_TYPE_SUPPORT}'
)
max_workers = max_workers or DEFAULT_MAX_WORKERS
if revision is None:
revision = DEFAULT_DATASET_REVISION if repo_type == REPO_TYPE_DATASET else DEFAULT_MODEL_REVISION

View File

@@ -82,6 +82,7 @@ class AigcModel:
trigger_words: Optional[List[str]] = None,
official_tags: Optional[List[str]] = None,
model_source: Optional[str] = 'USER_UPLOAD',
base_model_sub_type: Optional[str] = '',
):
"""
Initializes the AigcModel helper.
@@ -99,6 +100,7 @@ class AigcModel:
official_tags (List[str], optional): Official tags for the AIGC model. Defaults to None.
model_source (str, optional): Source of the model.
`USER_UPLOAD`, `TRAINED_FROM_MODELSCOPE` or `TRAINED_FROM_ALIYUN_FC`. Defaults to 'USER_UPLOAD'.
base_model_sub_type (str, Optional): Sub vision foundation model type. Defaults to ''. e.g. `SD_1_5`
"""
self.model_path = model_path
self.aigc_type = aigc_type
@@ -106,6 +108,7 @@ class AigcModel:
self.tag = tag
self.description = description
self.model_source = model_source
self.base_model_sub_type = base_model_sub_type
# Process cover images - convert local paths to base64 data URLs
if cover_images is not None:
processed_cover_images = []
@@ -391,6 +394,7 @@ class AigcModel:
'trigger_words': self.trigger_words,
'official_tags': self.official_tags,
'model_source': self.model_source,
'base_model_sub_type': self.base_model_sub_type,
}
@classmethod

View File

@@ -436,6 +436,25 @@ def _patch_pretrained_class(all_imported_modules, wrap=False):
**ignore_file_pattern_kwargs))
all_available_modules.append(var)
def get_class_from_dynamic_module(class_reference, *args, **kwargs):
from transformers.dynamic_module_utils import origin_get_class_from_dynamic_module
if '--' in class_reference:
repo_id, class_reference = class_reference.split('--')
if not os.path.exists(repo_id):
from modelscope import snapshot_download
repo_id = snapshot_download(repo_id)
class_reference = repo_id + '--' + class_reference
return origin_get_class_from_dynamic_module(class_reference, *args,
**kwargs)
from transformers import dynamic_module_utils
if not hasattr(dynamic_module_utils,
'origin_get_class_from_dynamic_module'):
dynamic_module_utils.origin_get_class_from_dynamic_module = dynamic_module_utils.get_class_from_dynamic_module
dynamic_module_utils.get_class_from_dynamic_module = get_class_from_dynamic_module
from transformers.models.auto import configuration_auto
configuration_auto.get_class_from_dynamic_module = get_class_from_dynamic_module
return all_available_modules
@@ -469,6 +488,13 @@ def _unpatch_pretrained_class(all_imported_modules):
except: # noqa
pass
from transformers import dynamic_module_utils
if hasattr(dynamic_module_utils, 'origin_get_class_from_dynamic_module'):
dynamic_module_utils.get_class_from_dynamic_module = dynamic_module_utils.origin_get_class_from_dynamic_module
from transformers.models.auto import configuration_auto
configuration_auto.get_class_from_dynamic_module = dynamic_module_utils.origin_get_class_from_dynamic_module
delattr(dynamic_module_utils, 'origin_get_class_from_dynamic_module')
def _patch_hub():
import huggingface_hub

View File

@@ -30,8 +30,6 @@ DEFAULT_IGNORE_PATTERNS = [
'*/.cache',
'**/.cache/**',
]
# Forbidden to commit these folders
FORBIDDEN_FOLDERS = ['.git', '.cache']
UploadMode = Literal['lfs', 'normal']
@@ -555,11 +553,7 @@ def _validate_path_in_repo(path_in_repo: str) -> str:
f"Invalid `path_in_repo` in CommitOperation: '{path_in_repo}'")
if path_in_repo.startswith('./'):
path_in_repo = path_in_repo[2:]
for forbidden in FORBIDDEN_FOLDERS:
if any(part == forbidden for part in path_in_repo.split('/')):
raise ValueError(
f"Invalid `path_in_repo` in CommitOperation: cannot update files under a '{forbidden}/' folder (path:"
f" '{path_in_repo}').")
return path_in_repo

View File

@@ -1,5 +1,5 @@
# Make sure to modify __release_datetime__ to release time when making official release.
__version__ = '1.32.0'
__version__ = '1.33.0'
# default release datetime for branches under active development is set
# to be a time far-far-away-into-the-future
__release_datetime__ = '2025-11-07 12:00:00'
__release_datetime__ = '2025-12-10 23:59:59'

View File

@@ -108,6 +108,9 @@ class HubRevisionTest(unittest.TestCase):
cache_dir=temp_cache_dir)
assert os.path.exists(file_path)
@unittest.skip(
'snapshot_download cannot exercise release-mode auto-selection while'
' DEFAULT_MODEL_REVISION forces a concrete branch (master).')
def test_snapshot_download_revision(self):
with mock.patch.dict(os.environ, self.modified_environ, clear=True):
self.prepare_repo_data_and_tag()
@@ -205,6 +208,7 @@ class HubRevisionTest(unittest.TestCase):
model_file_download(
self.model_id,
download_model_file_name2,
revision=None,
cache_dir=temp_cache_dir)
version.__release_datetime__ = t2
logger.info('Setting __release_datetime__ to: %s' % t2)