From a66907aecd958f5c3f5360042ab16f3f54e17bd0 Mon Sep 17 00:00:00 2001 From: Gongsheng Li <58078985+alcholiclg@users.noreply.github.com> Date: Tue, 11 Nov 2025 13:06:38 +0800 Subject: [PATCH 01/21] Feat/update default revision (#1546) * change DEFAULT_MODEL_REVISION to master * revert DEFAULT_MODEL_REVISION to None --------- Co-authored-by: alcholiclg --- modelscope/utils/constant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modelscope/utils/constant.py b/modelscope/utils/constant.py index a2897b9c..aef177bb 100644 --- a/modelscope/utils/constant.py +++ b/modelscope/utils/constant.py @@ -507,7 +507,7 @@ class Frameworks(object): REPO_TYPE_MODEL = 'model' REPO_TYPE_DATASET = 'dataset' REPO_TYPE_SUPPORT = [REPO_TYPE_MODEL, REPO_TYPE_DATASET] -DEFAULT_MODEL_REVISION = 'master' +DEFAULT_MODEL_REVISION = None MASTER_MODEL_BRANCH = 'master' DEFAULT_REPOSITORY_REVISION = 'master' DEFAULT_DATASET_REVISION = 'master' From debabe53a6ebf9f34cdcc1734ff4964770c7ef51 Mon Sep 17 00:00:00 2001 From: "Xingjun.Wang" Date: Tue, 18 Nov 2025 15:26:10 +0800 Subject: [PATCH 02/21] Fix default ignore patterns for upload_folder (#1549) * fix default ignore patterns for upload_folder * update repo_utils --- modelscope/hub/api.py | 5 ++--- modelscope/utils/repo_utils.py | 8 +------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 4cf0c02e..dabc97eb 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -2304,12 +2304,11 @@ class HubApi: allow_patterns = allow_patterns if allow_patterns else None ignore_patterns = ignore_patterns if ignore_patterns else None - # Ignore .git folder + # Ignore .git .cache folders if ignore_patterns is None: - ignore_patterns = [] + ignore_patterns = DEFAULT_IGNORE_PATTERNS elif isinstance(ignore_patterns, str): ignore_patterns = [ignore_patterns] - ignore_patterns += DEFAULT_IGNORE_PATTERNS commit_message = ( commit_message if commit_message is not None else f'Upload to {repo_id} on ModelScope hub' diff --git a/modelscope/utils/repo_utils.py b/modelscope/utils/repo_utils.py index f8f5cf71..fabe0a00 100644 --- a/modelscope/utils/repo_utils.py +++ b/modelscope/utils/repo_utils.py @@ -30,8 +30,6 @@ DEFAULT_IGNORE_PATTERNS = [ '*/.cache', '**/.cache/**', ] -# Forbidden to commit these folders -FORBIDDEN_FOLDERS = ['.git', '.cache'] UploadMode = Literal['lfs', 'normal'] @@ -555,11 +553,7 @@ def _validate_path_in_repo(path_in_repo: str) -> str: f"Invalid `path_in_repo` in CommitOperation: '{path_in_repo}'") if path_in_repo.startswith('./'): path_in_repo = path_in_repo[2:] - for forbidden in FORBIDDEN_FOLDERS: - if any(part == forbidden for part in path_in_repo.split('/')): - raise ValueError( - f"Invalid `path_in_repo` in CommitOperation: cannot update files under a '{forbidden}/' folder (path:" - f" '{path_in_repo}').") + return path_in_repo From ff87891acff102702d46fade28e536648837bd21 Mon Sep 17 00:00:00 2001 From: tastelikefeet <58414341+tastelikefeet@users.noreply.github.com> Date: Tue, 18 Nov 2025 16:28:38 +0800 Subject: [PATCH 03/21] fix exp type which affects hasattr (#1552) --- modelscope/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modelscope/__init__.py b/modelscope/__init__.py index e56d40ac..14057ef9 100644 --- a/modelscope/__init__.py +++ b/modelscope/__init__.py @@ -131,7 +131,7 @@ else: if module is not None: module = _patch_pretrained_class([module], wrap=True) else: - raise ImportError( + raise AttributeError( f'Cannot import available module of {name} in modelscope,' f' or related packages({hf_pkgs})') return module[0] From 609442d271bd7ed106a0933b1937289be7c1ad01 Mon Sep 17 00:00:00 2001 From: tastelikefeet <58414341+tastelikefeet@users.noreply.github.com> Date: Wed, 19 Nov 2025 11:15:56 +0800 Subject: [PATCH 04/21] Patch dynamic module (#1554) --- modelscope/utils/hf_util/patcher.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/modelscope/utils/hf_util/patcher.py b/modelscope/utils/hf_util/patcher.py index c6c0527b..c0ebdb22 100644 --- a/modelscope/utils/hf_util/patcher.py +++ b/modelscope/utils/hf_util/patcher.py @@ -436,6 +436,25 @@ def _patch_pretrained_class(all_imported_modules, wrap=False): **ignore_file_pattern_kwargs)) all_available_modules.append(var) + + def get_class_from_dynamic_module(class_reference, *args, **kwargs): + from transformers.dynamic_module_utils import origin_get_class_from_dynamic_module + if '--' in class_reference: + repo_id, class_reference = class_reference.split('--') + if not os.path.exists(repo_id): + from modelscope import snapshot_download + repo_id = snapshot_download(repo_id) + class_reference = repo_id + '--' + class_reference + return origin_get_class_from_dynamic_module(class_reference, *args, + **kwargs) + + from transformers import dynamic_module_utils + if not hasattr(dynamic_module_utils, + 'origin_get_class_from_dynamic_module'): + dynamic_module_utils.origin_get_class_from_dynamic_module = dynamic_module_utils.get_class_from_dynamic_module + dynamic_module_utils.get_class_from_dynamic_module = get_class_from_dynamic_module + from transformers.models.auto import configuration_auto + configuration_auto.get_class_from_dynamic_module = get_class_from_dynamic_module return all_available_modules @@ -469,6 +488,13 @@ def _unpatch_pretrained_class(all_imported_modules): except: # noqa pass + from transformers import dynamic_module_utils + if hasattr(dynamic_module_utils, 'origin_get_class_from_dynamic_module'): + dynamic_module_utils.get_class_from_dynamic_module = dynamic_module_utils.origin_get_class_from_dynamic_module + from transformers.models.auto import configuration_auto + configuration_auto.get_class_from_dynamic_module = dynamic_module_utils.origin_get_class_from_dynamic_module + delattr(dynamic_module_utils, 'origin_get_class_from_dynamic_module') + def _patch_hub(): import huggingface_hub From 12988976a3f1c66050603dd8edb32075fceb1c31 Mon Sep 17 00:00:00 2001 From: "Xingjun.Wang" Date: Thu, 20 Nov 2025 11:23:58 +0800 Subject: [PATCH 05/21] add new args for aigc create_model_tag (#1555) --- modelscope/hub/api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index dabc97eb..3b65e471 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -394,7 +394,9 @@ class HubApi: 'WeightsName': aigc_model.weight_filename, 'WeightsSha256': aigc_model.weight_sha256, 'WeightsSize': aigc_model.weight_size, - 'TriggerWords': aigc_model.trigger_words + 'TriggerWords': aigc_model.trigger_words, + 'AigcType': aigc_model.aigc_type, + 'VisionFoundation': aigc_model.base_model_type } else: From 6f3200dfc277c03817c80c70178a410e350ebe2a Mon Sep 17 00:00:00 2001 From: baymax591 Date: Wed, 26 Nov 2025 09:56:47 +0800 Subject: [PATCH 06/21] [docker] Add Ascend NPU dockerfile (#1550) Co-authored-by: ji-huazhong Co-authored-by: vx120 <893600387@qq.com> --- docker/Dockerfile.ascend | 120 +++++++++++++++++++++++++++++++++++++++ docker/build_image.py | 34 +++++++++++ 2 files changed, 154 insertions(+) create mode 100644 docker/Dockerfile.ascend diff --git a/docker/Dockerfile.ascend b/docker/Dockerfile.ascend new file mode 100644 index 00000000..ce8db144 --- /dev/null +++ b/docker/Dockerfile.ascend @@ -0,0 +1,120 @@ +FROM {base_image} + +RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple && \ + pip config set install.trusted-host mirrors.aliyun.com + +# Prepare required system dependencies +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends gcc g++ cmake libnuma-dev wget git curl jq vim build-essential && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ + pip install --upgrade pip setuptools packaging && \ + pip cache purge + +{extra_content} + +# Prepare repositories with low update frequency +RUN ARCH=$(uname -m) && \ + # Set extra pip index for x86_64 platform + echo "[LOG INFO] Detected architecture: $ARCH" && \ + if [ "$ARCH" = "x86_64" ]; then \ + pip config set global.extra-index-url "https://download.pytorch.org/whl/cpu/"; \ + fi && \ + # Clone libs + git clone --depth 1 --branch v0.11.0 https://github.com/vllm-project/vllm && \ + git clone --depth 1 --branch v0.11.0rc1 https://github.com/vllm-project/vllm-ascend.git && \ + git clone https://gitcode.com/Ascend/MindSpeed.git && \ + cd MindSpeed && git checkout f2b0977e && cd .. && \ + git clone --depth 1 --branch core_v0.12.1 https://github.com/NVIDIA/Megatron-LM.git + +# Install repositories with low update frequency +RUN ARCH=$(uname -m) && \ + # Export and source env + if [ "$ARCH" = "aarch64" ]; then \ + export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/8.3.RC1/aarch64-linux/devlib/linux/aarch64:$LD_LIBRARY_PATH; \ + elif [ "$ARCH" = "x86_64" ]; then \ + export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/8.3.RC1/x86_64-linux/devlib/linux/x86_64/:$LD_LIBRARY_PATH; \ + fi && \ + source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ + source /usr/local/Ascend/nnal/atb/set_env.sh && \ + # Install torch & torch_npu & torchvision + pip install torch==2.7.1 torch_npu==2.7.1 torchvision==0.22.1 && \ + # Install vllm + cd vllm && VLLM_TARGET_DEVICE=empty pip install -v -e . && cd .. && \ + # Install vllm-ascend + cd vllm-ascend && pip install -v -e . && cd .. && \ + # Install MindSpeed & Megatron + pip install -e MindSpeed && \ + # Clear extra files + rm -rf /tmp/* /var/tmp/* && \ + pip cache purge + +ENV PYTHONPATH ${PYTHONPATH}:/Megatron-LM + +ARG INSTALL_MS_DEPS={install_ms_deps} + +# install dependencies +COPY requirements /var/modelscope + +RUN pip uninstall ms-swift modelscope -y && pip --no-cache-dir install pip==23.* -U && \ +if [ "$INSTALL_MS_DEPS" = "True" ]; then \ + pip --no-cache-dir install omegaconf==2.0.6 && \ + pip install 'editdistance==0.8.1' && \ + pip install --no-cache-dir 'cython<=0.29.36' versioneer 'numpy<2.0' -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ + pip install --no-cache-dir -r /var/modelscope/framework.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ + pip install --no-cache-dir -r /var/modelscope/audio.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ + pip install --no-cache-dir -r /var/modelscope/cv.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ + pip install --no-cache-dir -r /var/modelscope/multi-modal.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ + pip install --no-cache-dir -r /var/modelscope/nlp.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ + pip install --no-cache-dir -r /var/modelscope/science.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ + pip install --no-cache-dir -r /var/modelscope/tests.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ + pip install --no-cache-dir -r /var/modelscope/server.txt && \ + pip install --no-cache-dir https://modelscope.oss-cn-beijing.aliyuncs.com/packages/imageio_ffmpeg-0.4.9-py3-none-any.whl --no-dependencies --force && \ + pip install adaseq pai-easycv && \ + pip install --no-cache-dir 'scipy<1.13.0' && \ + pip install --no-cache-dir funtextprocessing typeguard==2.13.3 scikit-learn -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ + pip install --no-cache-dir text2sql_lgesql==1.3.0 git+https://github.com/jin-s13/xtcocoapi.git@v1.14 git+https://github.com/gatagat/lap.git@v0.4.0 -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --force --no-deps && \ + pip install --no-cache-dir mmcls>=0.21.0 mmdet>=2.25.0 decord>=0.6.0 mpi4py paint_ldm ipykernel fasttext -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ + pip uninstall ddpm_guided_diffusion -y && \ + pip install --no-cache-dir 'blobfile>=1.0.5' && \ + pip install 'ddpm_guided_diffusion' -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --no-index && \ + pip uninstall shotdetect_scenedetect_lgss -y && \ + pip install 'shotdetect_scenedetect_lgss' -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --no-index && \ + pip uninstall MinDAEC -y && \ + pip install https://modelscope.oss-cn-beijing.aliyuncs.com/releases/dependencies/MinDAEC-0.0.2-py3-none-any.whl && \ + pip cache purge; \ +else \ + pip install --no-cache-dir -r /var/modelscope/framework.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ + pip cache purge; \ +fi + +ARG CUR_TIME={cur_time} +RUN echo $CUR_TIME + +RUN curl -fsSL https://ollama.com/install.sh | sh && \ + pip install --no-cache-dir -U funasr scikit-learn && \ + pip install --no-cache-dir -U qwen_vl_utils qwen_omni_utils librosa timm transformers accelerate peft trl safetensors && \ + cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b {swift_branch} --single-branch https://github.com/modelscope/ms-swift.git && \ + cd ms-swift && pip install .[llm] && \ + pip install .[eval] && pip install evalscope -U --no-dependencies && pip install ms-agent -U --no-dependencies && \ + cd / && rm -fr /tmp/ms-swift && pip cache purge; \ + cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b {modelscope_branch} --single-branch https://github.com/modelscope/modelscope.git && \ + cd modelscope && pip install . -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ + cd / && rm -fr /tmp/modelscope && pip cache purge; \ + pip install --no-cache-dir transformers diffusers timm>=0.9.0 && pip cache purge; \ + pip install --no-cache-dir omegaconf==2.3.0 && pip cache purge; + + +RUN if [ "$INSTALL_MS_DEPS" = "True" ]; then \ + pip install --no-cache-dir huggingface-hub transformers peft -U; \ +fi; + +ENV SETUPTOOLS_USE_DISTUTILS=stdlib +ENV VLLM_USE_MODELSCOPE=True +ENV LMDEPLOY_USE_MODELSCOPE=True +ENV MODELSCOPE_CACHE=/mnt/workspace/.cache/modelscope/hub + +# Show install results +RUN pip list + +SHELL ["/bin/bash", "-c"] diff --git a/docker/build_image.py b/docker/build_image.py index 07c616f4..916bb5c2 100644 --- a/docker/build_image.py +++ b/docker/build_image.py @@ -415,6 +415,38 @@ RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy return os.system(f'docker push {image_tag2}') +class AscendSwiftImageBuilder(SwiftImageBuilder): + def init_args(self, args) -> Any: + if not args.base_image: + # other vision search for: https://hub.docker.com/r/ascendai/cann/tags + args.base_image = "swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:8.3.rc1-a3-ubuntu22.04-py3.11" + return super().init_args(args) + + def generate_dockerfile(self) -> str: + extra_content = """ +RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy +""" + with open('docker/Dockerfile.ascend', 'r') as f: + content = f.read() + content = content.replace('{base_image}', self.args.base_image) + content = content.replace('{extra_content}', extra_content) + content = content.replace('{cur_time}', formatted_time) + content = content.replace('{install_ms_deps}', 'False') + content = content.replace('{modelscope_branch}', + self.args.modelscope_branch) + content = content.replace('{swift_branch}', self.args.swift_branch) + return content + + def image(self) -> str: + return ( + f'{docker_registry}:{self.args.base_image.split(":")[-1]}-torch2.7.1' + f'-{self.args.modelscope_version}-ascend-swift-test' + ) + + def push(self): + return 0 + + parser = argparse.ArgumentParser() parser.add_argument('--base_image', type=str, default=None) parser.add_argument('--image_type', type=str) @@ -448,6 +480,8 @@ elif args.image_type.lower() == 'llm': builder_cls = LLMImageBuilder elif args.image_type.lower() == 'swift': builder_cls = SwiftImageBuilder +elif args.image_type.lower() == 'ascend_swift': + builder_cls = AscendSwiftImageBuilder else: raise ValueError(f'Unsupported image_type: {args.image_type}') From 9cb7ce7029ed8deddf863507eafbc3850d2cafc0 Mon Sep 17 00:00:00 2001 From: Yunlin Mao Date: Tue, 2 Dec 2025 10:50:39 +0800 Subject: [PATCH 07/21] Update build file and workflow (#1551) * update build file * update setup file * update fairseq dep * fix error log --- .github/workflows/daily_regression.yaml | 1 - .github/workflows/publish.yaml | 3 +-- Makefile | 3 ++- docker/Dockerfile.ubuntu | 15 ++++++++------- docker/build_image.py | 22 ++++++++++++---------- docker/install.sh | 2 ++ modelscope/hub/errors.py | 9 +++++---- pyproject.toml | 11 ++++++++--- requirements/multi-modal.txt | 2 +- 9 files changed, 39 insertions(+), 29 deletions(-) diff --git a/.github/workflows/daily_regression.yaml b/.github/workflows/daily_regression.yaml index 73d971e6..bf73d3ea 100644 --- a/.github/workflows/daily_regression.yaml +++ b/.github/workflows/daily_regression.yaml @@ -32,7 +32,6 @@ jobs: - name: Fetch LFS objects run: | - git lfs install --local --force git lfs pull - name: Run unittest diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 0eadd339..49972e1f 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -24,8 +24,7 @@ jobs: - name: Build ModelScope # Build AST template before packaging run: | - python -c "from modelscope.utils.ast_utils import generate_ast_template; generate_ast_template()" - python setup.py sdist bdist_wheel + make whl - name: Publish package to PyPI run: | pip install twine diff --git a/Makefile b/Makefile index 96532199..92385886 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,8 @@ test: .PHONY: whl whl: - python setup.py sdist bdist_wheel + python -c "from modelscope.utils.ast_utils import generate_ast_template; generate_ast_template()" + python setup.py sdist --dist-dir $(WHL_BUILD_DIR)/dist bdist_wheel --dist-dir $(WHL_BUILD_DIR)/dist .PHONY: clean clean: diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu index e889c8d7..ec5e70a1 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.ubuntu @@ -16,7 +16,7 @@ COPY {meta_file} /tmp/install.sh ARG INSTALL_MS_DEPS={install_ms_deps} -ARG INSTALL_MEGATRON_DEPS={install_megatron_deps} +ARG IMAGE_TYPE={image_type} # install dependencies COPY requirements /var/modelscope @@ -56,7 +56,7 @@ fi ARG CUR_TIME={cur_time} RUN echo $CUR_TIME -RUN sh /tmp/install.sh {version_args} && \ +RUN bash /tmp/install.sh {version_args} && \ curl -fsSL https://ollama.com/install.sh | sh && \ pip install --no-cache-dir -U funasr scikit-learn && \ pip install --no-cache-dir -U qwen_vl_utils qwen_omni_utils librosa timm transformers accelerate peft trl safetensors && \ @@ -75,10 +75,7 @@ RUN sh /tmp/install.sh {version_args} && \ cp /tmp/resources/ubuntu2204.aliyun /etc/apt/sources.list -RUN if [ "$INSTALL_MS_DEPS" = "True" ]; then \ - pip install --no-cache-dir huggingface-hub transformers peft -U; \ -fi; \ -if [ "$INSTALL_MEGATRON_DEPS" = "True" ]; then \ +RUN if [ "$IMAGE_TYPE" = "swift" ]; then \ pip install "sglang[all]<0.5" "math_verify==0.5.2" "gradio<5.33" -U && \ pip install liger_kernel wandb swanlab nvitop pre-commit "transformers<4.57" "trl<0.21" huggingface-hub -U && \ SITE_PACKAGES=$(python -c "import site; print(site.getsitepackages()[0])") && echo $SITE_PACKAGES && \ @@ -88,6 +85,11 @@ if [ "$INSTALL_MEGATRON_DEPS" = "True" ]; then \ cd apex && git checkout e13873debc4699d39c6861074b9a3b2a02327f92 && pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" ./ && \ cd / && rm -fr /tmp/apex && pip cache purge; \ pip install git+https://github.com/NVIDIA/Megatron-LM.git@core_r0.13.0; \ +elif [ "$IMAGE_TYPE" = "llm" ]; then \ + pip install --no-cache-dir huggingface-hub transformers peft diffusers -U; \ + pip uninstall autoawq -y; \ +else \ + pip install "transformers<4.56" "tokenizers<0.22" "trl<0.23" "diffusers<0.35" --no-dependencies; \ fi # install nvm and set node version to 18 @@ -97,7 +99,6 @@ RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.3/install.sh | b nvm install 22 && \ nvm use 22 -ENV SETUPTOOLS_USE_DISTUTILS=stdlib ENV VLLM_USE_MODELSCOPE=True ENV LMDEPLOY_USE_MODELSCOPE=True ENV MODELSCOPE_CACHE=/mnt/workspace/.cache/modelscope/hub diff --git a/docker/build_image.py b/docker/build_image.py index 916bb5c2..4cc4075a 100644 --- a/docker/build_image.py +++ b/docker/build_image.py @@ -162,7 +162,7 @@ class CPUImageBuilder(Builder): content = content.replace('{version_args}', version_args) content = content.replace('{cur_time}', formatted_time) content = content.replace('{install_ms_deps}', 'True') - content = content.replace('{install_megatron_deps}', 'False') + content = content.replace('{image_type}', 'cpu') content = content.replace('{torch_version}', self.args.torch_version) content = content.replace('{torchvision_version}', @@ -207,6 +207,8 @@ class GPUImageBuilder(Builder): # pushd ~ popd is to solve the tf cannot use gpu problem. extra_content = """ RUN pip install tf-keras==2.16.0 --no-dependencies && \ + pip install onnx==1.18.0 --no-dependencies && \ + pip install deepspeed==0.17.4 --no-dependencies && \ pip install --no-cache-dir torchsde jupyterlab torchmetrics==0.11.4 basicsr pynvml shortuuid && \ CUDA_HOME=/usr/local/cuda TORCH_CUDA_ARCH_LIST="6.0 6.1 7.0 7.5 8.0 8.6 8.9 9.0" \ pip install --no-cache-dir 'git+https://github.com/facebookresearch/detectron2.git' @@ -230,7 +232,7 @@ RUN pushd $(dirname $(python -c 'print(__import__("tensorflow").__file__)')) && content = content.replace('{version_args}', version_args) content = content.replace('{cur_time}', formatted_time) content = content.replace('{install_ms_deps}', 'True') - content = content.replace('{install_megatron_deps}', 'False') + content = content.replace('{image_type}', 'gpu') content = content.replace('{torch_version}', self.args.torch_version) content = content.replace('{torchvision_version}', @@ -273,19 +275,19 @@ class LLMImageBuilder(Builder): # A mirrored image of nvidia/cuda:12.4.0-devel-ubuntu22.04 args.base_image = 'nvidia/cuda:12.4.0-devel-ubuntu22.04' if not args.torch_version: - args.torch_version = '2.6.0' - args.torchaudio_version = '2.6.0' - args.torchvision_version = '0.21.0' + args.torch_version = '2.8.0' + args.torchaudio_version = '2.8.0' + args.torchvision_version = '0.23.0' if not args.cuda_version: args.cuda_version = '12.4.0' if not args.vllm_version: - args.vllm_version = '0.8.5.post1' + args.vllm_version = '0.11.0' if not args.lmdeploy_version: - args.lmdeploy_version = '0.9.1' + args.lmdeploy_version = '0.10.1' if not args.autogptq_version: args.autogptq_version = '0.7.1' if not args.flashattn_version: - args.flashattn_version = '2.7.1.post4' + args.flashattn_version = '2.7.4.post1' return args def generate_dockerfile(self) -> str: @@ -306,7 +308,7 @@ class LLMImageBuilder(Builder): content = content.replace('{version_args}', version_args) content = content.replace('{cur_time}', formatted_time) content = content.replace('{install_ms_deps}', 'False') - content = content.replace('{install_megatron_deps}', 'False') + content = content.replace('{image_type}', 'llm') content = content.replace('{torch_version}', self.args.torch_version) content = content.replace('{torchvision_version}', @@ -382,7 +384,7 @@ RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy content = content.replace('{version_args}', version_args) content = content.replace('{cur_time}', formatted_time) content = content.replace('{install_ms_deps}', 'False') - content = content.replace('{install_megatron_deps}', 'True') + content = content.replace('{image_type}', 'swift') content = content.replace('{torch_version}', self.args.torch_version) content = content.replace('{torchvision_version}', diff --git a/docker/install.sh b/docker/install.sh index b15dab31..661dcc81 100644 --- a/docker/install.sh +++ b/docker/install.sh @@ -28,6 +28,8 @@ pip install --no-cache-dir triton auto-gptq==$autogptq_version -U && pip cache p if [[ "$(printf '%s\n' "0.6.0" "$vllm_version" | sort -V | head -n1)" = "0.6.0" ]]; then # vllm_version is >= 0.6.0 pip install --no-cache-dir vllm==$vllm_version && pip cache purge +else + echo "vllm_version < 0.6.0, skipping installation. (vllm_version = $vllm_version)" fi # pip uninstall -y torch-scatter && TORCH_CUDA_ARCH_LIST="6.0;6.1;6.2;7.0;7.5;8.0;8.6;8.9;9.0" pip install --no-cache-dir -U torch-scatter diff --git a/modelscope/hub/errors.py b/modelscope/hub/errors.py index 2f63edd9..e5fb3621 100644 --- a/modelscope/hub/errors.py +++ b/modelscope/hub/errors.py @@ -120,11 +120,12 @@ def handle_http_response(response: requests.Response, http_error_msg = 'The request model: %s does not exist!' % (model_id) elif 403 == response.status_code: if cookies is None: - http_error_msg = 'Authentication token does not exist, ' - 'failed to access model {model_id} which may not exist or may be ' - 'private. Please login first.' + http_error_msg = ( + f'Authentication token does not exist, failed to access model {model_id} ' + 'which may not exist or may be private. Please login first.') + else: - http_error_msg = 'The authentication token is invalid, failed to access model {model_id}.' + http_error_msg = f'The authentication token is invalid, failed to access model {model_id}.' elif 400 <= response.status_code < 500: http_error_msg = u'%s Client Error: %s, Request id: %s for url: %s' % ( response.status_code, reason, request_id, response.url) diff --git a/pyproject.toml b/pyproject.toml index d3036994..3b95faac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,8 @@ hub = {file = ["requirements/hub.txt"]} datasets = {file = ["requirements/datasets.txt"]} framework = {file = ["requirements/framework.txt"]} server = {file = ["requirements/server.txt"]} +docs = {file = ["requirements/docs.txt"]} +tests = {file = ["requirements/tests.txt"]} # domain specific with framework base cv = {file = ["requirements/framework.txt", "requirements/cv.txt"]} @@ -59,9 +61,12 @@ audio_codec = {file = ["requirements/framework.txt", "requirements/audio/audio audio_tts = {file = ["requirements/framework.txt", "requirements/audio/audio_tts.txt"]} audio_kws = {file = ["requirements/framework.txt", "requirements/audio/audio_kws.txt"]} audio_signal = {file = ["requirements/framework.txt", "requirements/audio/audio_signal.txt"]} - -docs = {file = ["requirements/docs.txt"]} -tests = {file = ["requirements/tests.txt"]} +audio = {file = ["requirements/framework.txt", + "requirements/audio/audio_asr.txt", + "requirements/audio/audio_codec.txt", + "requirements/audio/audio_tts.txt", + "requirements/audio/audio_kws.txt", + "requirements/audio/audio_signal.txt"]} # skip audio requirements due to its hard dependency which may cause installation failure all = {file = [ diff --git a/requirements/multi-modal.txt b/requirements/multi-modal.txt index af12af06..46ac0054 100644 --- a/requirements/multi-modal.txt +++ b/requirements/multi-modal.txt @@ -5,7 +5,7 @@ diffusers>=0.25.0 # 0.12.1 has issue of No such file or directory: 'fairseq/version.txt' # 0.12.2 not support py311 #fairseq==0.12.2 -fairseq @ https://github.com/liyaodev/fairseq/releases/download/v0.12.3.1/fairseq-0.12.3.1-cp311-cp311-linux_x86_64.whl ; python_version == "3.11" and platform_system == "Linux" and platform_machine == "x86_64" +fairseq-fixed==0.12.3.1 ftfy>=6.0.3 librosa==0.10.1 opencv-python From f35932e90b5c5b10ae58989cacfaf88130b3e781 Mon Sep 17 00:00:00 2001 From: suluyan Date: Fri, 5 Dec 2025 11:20:12 +0800 Subject: [PATCH 08/21] bump version --- modelscope/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modelscope/version.py b/modelscope/version.py index 031a86b4..698d3dbb 100644 --- a/modelscope/version.py +++ b/modelscope/version.py @@ -1,5 +1,5 @@ # Make sure to modify __release_datetime__ to release time when making official release. -__version__ = '2.0.0' +__version__ = '1.33.0' # default release datetime for branches under active development is set # to be a time far-far-away-into-the-future -__release_datetime__ = '2099-09-06 00:00:00' +__release_datetime__ = '2025-12-10 23:59:59' From a31c44cb5a5c59635cfc1f69a143f8101e1dd756 Mon Sep 17 00:00:00 2001 From: Gongsheng Li <58078985+alcholiclg@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:24:42 +0800 Subject: [PATCH 09/21] Feat/update default revision (#1553) --- modelscope/utils/constant.py | 2 +- tests/hub/test_hub_revision_release_mode.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/modelscope/utils/constant.py b/modelscope/utils/constant.py index aef177bb..a2897b9c 100644 --- a/modelscope/utils/constant.py +++ b/modelscope/utils/constant.py @@ -507,7 +507,7 @@ class Frameworks(object): REPO_TYPE_MODEL = 'model' REPO_TYPE_DATASET = 'dataset' REPO_TYPE_SUPPORT = [REPO_TYPE_MODEL, REPO_TYPE_DATASET] -DEFAULT_MODEL_REVISION = None +DEFAULT_MODEL_REVISION = 'master' MASTER_MODEL_BRANCH = 'master' DEFAULT_REPOSITORY_REVISION = 'master' DEFAULT_DATASET_REVISION = 'master' diff --git a/tests/hub/test_hub_revision_release_mode.py b/tests/hub/test_hub_revision_release_mode.py index 74a48527..a3cf8cdf 100644 --- a/tests/hub/test_hub_revision_release_mode.py +++ b/tests/hub/test_hub_revision_release_mode.py @@ -108,6 +108,9 @@ class HubRevisionTest(unittest.TestCase): cache_dir=temp_cache_dir) assert os.path.exists(file_path) + @unittest.skip( + 'snapshot_download cannot exercise release-mode auto-selection while' + ' DEFAULT_MODEL_REVISION forces a concrete branch (master).') def test_snapshot_download_revision(self): with mock.patch.dict(os.environ, self.modified_environ, clear=True): self.prepare_repo_data_and_tag() @@ -205,6 +208,7 @@ class HubRevisionTest(unittest.TestCase): model_file_download( self.model_id, download_model_file_name2, + revision=None, cache_dir=temp_cache_dir) version.__release_datetime__ = t2 logger.info('Setting __release_datetime__ to: %s' % t2) From be6874ea8e3ed2830cd370b7bfca2224f1fbeaef Mon Sep 17 00:00:00 2001 From: "Xingjun.Wang" Date: Fri, 5 Dec 2025 15:37:45 +0800 Subject: [PATCH 10/21] Feat: add set_repo_visibility in hub api (#1564) * add set_repo_visibility * fix cr * fix lint --- docker/build_image.py | 6 ++-- modelscope/hub/api.py | 79 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/docker/build_image.py b/docker/build_image.py index 4cc4075a..22fc21bd 100644 --- a/docker/build_image.py +++ b/docker/build_image.py @@ -418,10 +418,11 @@ RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy class AscendSwiftImageBuilder(SwiftImageBuilder): + def init_args(self, args) -> Any: if not args.base_image: # other vision search for: https://hub.docker.com/r/ascendai/cann/tags - args.base_image = "swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:8.3.rc1-a3-ubuntu22.04-py3.11" + args.base_image = 'swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:8.3.rc1-a3-ubuntu22.04-py3.11' return super().init_args(args) def generate_dockerfile(self) -> str: @@ -442,8 +443,7 @@ RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy def image(self) -> str: return ( f'{docker_registry}:{self.args.base_image.split(":")[-1]}-torch2.7.1' - f'-{self.args.modelscope_version}-ascend-swift-test' - ) + f'-{self.args.modelscope_version}-ascend-swift-test') def push(self): return 0 diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 3b65e471..1a8ccc5d 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -19,7 +19,8 @@ from http import HTTPStatus from http.cookiejar import CookieJar from os.path import expanduser from pathlib import Path -from typing import Any, BinaryIO, Dict, Iterable, List, Optional, Tuple, Union +from typing import (Any, BinaryIO, Dict, Iterable, List, Literal, Optional, + Tuple, Union) from urllib.parse import urlencode import json @@ -457,7 +458,7 @@ class HubApi: model_id: str, revision: Optional[str] = DEFAULT_MODEL_REVISION, endpoint: Optional[str] = None - ) -> str: + ) -> dict: """Get model information at ModelScope Args: @@ -2868,6 +2869,80 @@ class HubApi: 'total_files': len(to_delete) } + def set_repo_visibility(self, + repo_id: str, + repo_type: Literal['model', 'dataset'], + visibility: Literal['private', 'public'], + token: Union[str, None] = None + ) -> dict: + """ + Set the visibility of a repo. + + Args: + repo_id (str): The repo id in the format of `owner_name/repo_name`. + repo_type (Literal['model', 'dataset']): The repo type, `model` or `dataset`. + visibility (Literal['private', 'public']): The visibility to set, `private` or `public`. + token (Union[str, None]): The access token. If None, will use the cookies from the local cache. + See `https://modelscope.cn/my/myaccesstoken` to get your token. + + Returns: + dict: The response from the server. + """ + if not repo_id: + raise ValueError('The arg `repo_id` cannot be empty!') + + visibility_map: Dict[str, int] = {v: k for k, v in VisibilityMap.items()} + visibility_code: int = visibility_map.get(visibility, 5) + cookies = self.get_cookies(access_token=token, cookies_required=True) + + if repo_type == REPO_TYPE_MODEL: + model_info = self.get_model(model_id=repo_id) + path = f'{self.endpoint}/api/v1/models/{repo_id}' + payload = { + 'ChineseName': model_info.get('ChineseName', ''), + 'ModelFramework': model_info.get('ModelFramework', 'Pytorch'), + 'Visibility': visibility_code, + 'ProtectedMode': 2, + 'ApprovalMode': model_info.get('ApprovalMode', 2), + 'Description': model_info.get('Description', ''), + 'AigcType': model_info.get('AigcType', ''), + 'VisionFoundation': model_info.get('VisionFoundation', ''), + 'ModelCover': model_info.get('ModelCover', ''), + 'SubScientificField': model_info.get('SubScientificField', None), + 'ScientificField': model_info.get('NEXA', {}).get('ScientificField', ''), + 'Source': model_info.get('NEXA', {}).get('Source', ''), + } + elif repo_type == REPO_TYPE_DATASET: + + repo_id_parts = repo_id.split('/') + if len(repo_id_parts) != 2 or not all(repo_id_parts): + raise ValueError(f'Invalid dataset repo_id: {repo_id}, should be in format of `owner/dataset_name`') + + dataset_idx, _ = self.get_dataset_id_and_type( + dataset_name=repo_id_parts[1], + namespace=repo_id_parts[0], + ) + + path = f'{self.endpoint}/api/v1/datasets/{dataset_idx}' + payload = { + 'Visibility': visibility_code, + 'ProtectedMode': 2, + } + else: + raise ValueError(f'Invalid repo type: {repo_type}, supported repos: {REPO_TYPE_SUPPORT}') + + r = self.session.put( + path, + json=payload, + cookies=cookies, + headers=self.builder_headers(self.headers)) + + raise_for_http_status(r) + resp = r.json() + raise_on_error(resp) + + return resp + class ModelScopeConfig: path_credential = expanduser(MODELSCOPE_CREDENTIALS_PATH) From 7543a9c7d2ed56f75110ec01cdd4cb4afe15fdc3 Mon Sep 17 00:00:00 2001 From: "Xingjun.Wang" Date: Fri, 5 Dec 2025 15:41:22 +0800 Subject: [PATCH 11/21] Add base_model_sub_type for aigc model (#1563) --- modelscope/cli/create.py | 6 ++++++ modelscope/hub/api.py | 1 + modelscope/hub/utils/aigc.py | 4 ++++ 3 files changed, 11 insertions(+) diff --git a/modelscope/cli/create.py b/modelscope/cli/create.py index f9e9e48c..b6048532 100644 --- a/modelscope/cli/create.py +++ b/modelscope/cli/create.py @@ -129,6 +129,11 @@ class CreateCMD(CLICommand): help= 'Source of the AIGC model. `USER_UPLOAD`, `TRAINED_FROM_MODELSCOPE` or `TRAINED_FROM_ALIYUN_FC`.' ) + aigc_group.add_argument( + '--base_model_sub_type', + type=str, + default='', + help='Base model sub type, e.g., Qwen_Edit_2509') parser.set_defaults(func=subparser_func) @@ -191,6 +196,7 @@ class CreateCMD(CLICommand): base_model_id=self.args.base_model_id, path_in_repo=self.args.path_in_repo, model_source=self.args.model_source, + base_model_sub_type=self.args.base_model_sub_type, ) # Convert visibility string to int for the API call diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 1a8ccc5d..13cf53fb 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -305,6 +305,7 @@ class HubApi: 'ModelPath': aigc_model.model_path, 'TriggerWords': aigc_model.trigger_words, 'ModelSource': aigc_model.model_source, + 'SubVisionFoundation': aigc_model.base_model_sub_type, }) if aigc_model.official_tags: diff --git a/modelscope/hub/utils/aigc.py b/modelscope/hub/utils/aigc.py index 62df6cf8..2f2644ef 100644 --- a/modelscope/hub/utils/aigc.py +++ b/modelscope/hub/utils/aigc.py @@ -82,6 +82,7 @@ class AigcModel: trigger_words: Optional[List[str]] = None, official_tags: Optional[List[str]] = None, model_source: Optional[str] = 'USER_UPLOAD', + base_model_sub_type: Optional[str] = '', ): """ Initializes the AigcModel helper. @@ -99,6 +100,7 @@ class AigcModel: official_tags (List[str], optional): Official tags for the AIGC model. Defaults to None. model_source (str, optional): Source of the model. `USER_UPLOAD`, `TRAINED_FROM_MODELSCOPE` or `TRAINED_FROM_ALIYUN_FC`. Defaults to 'USER_UPLOAD'. + base_model_sub_type (str, Optional): Sub vision foundation model type. Defaults to ''. e.g. `SD_1_5` """ self.model_path = model_path self.aigc_type = aigc_type @@ -106,6 +108,7 @@ class AigcModel: self.tag = tag self.description = description self.model_source = model_source + self.base_model_sub_type = base_model_sub_type # Process cover images - convert local paths to base64 data URLs if cover_images is not None: processed_cover_images = [] @@ -391,6 +394,7 @@ class AigcModel: 'trigger_words': self.trigger_words, 'official_tags': self.official_tags, 'model_source': self.model_source, + 'base_model_sub_type': self.base_model_sub_type, } @classmethod From c2f91d80fedaa745f42c44577a0937e04d372d2f Mon Sep 17 00:00:00 2001 From: Gongsheng Li <58078985+alcholiclg@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:24:42 +0800 Subject: [PATCH 12/21] Feat/update default revision (#1553) --- modelscope/utils/constant.py | 2 +- tests/hub/test_hub_revision_release_mode.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/modelscope/utils/constant.py b/modelscope/utils/constant.py index aef177bb..a2897b9c 100644 --- a/modelscope/utils/constant.py +++ b/modelscope/utils/constant.py @@ -507,7 +507,7 @@ class Frameworks(object): REPO_TYPE_MODEL = 'model' REPO_TYPE_DATASET = 'dataset' REPO_TYPE_SUPPORT = [REPO_TYPE_MODEL, REPO_TYPE_DATASET] -DEFAULT_MODEL_REVISION = None +DEFAULT_MODEL_REVISION = 'master' MASTER_MODEL_BRANCH = 'master' DEFAULT_REPOSITORY_REVISION = 'master' DEFAULT_DATASET_REVISION = 'master' diff --git a/tests/hub/test_hub_revision_release_mode.py b/tests/hub/test_hub_revision_release_mode.py index 74a48527..a3cf8cdf 100644 --- a/tests/hub/test_hub_revision_release_mode.py +++ b/tests/hub/test_hub_revision_release_mode.py @@ -108,6 +108,9 @@ class HubRevisionTest(unittest.TestCase): cache_dir=temp_cache_dir) assert os.path.exists(file_path) + @unittest.skip( + 'snapshot_download cannot exercise release-mode auto-selection while' + ' DEFAULT_MODEL_REVISION forces a concrete branch (master).') def test_snapshot_download_revision(self): with mock.patch.dict(os.environ, self.modified_environ, clear=True): self.prepare_repo_data_and_tag() @@ -205,6 +208,7 @@ class HubRevisionTest(unittest.TestCase): model_file_download( self.model_id, download_model_file_name2, + revision=None, cache_dir=temp_cache_dir) version.__release_datetime__ = t2 logger.info('Setting __release_datetime__ to: %s' % t2) From 4890a8b52713e91027b245a20cd088692b51a638 Mon Sep 17 00:00:00 2001 From: "Xingjun.Wang" Date: Fri, 5 Dec 2025 15:37:45 +0800 Subject: [PATCH 13/21] Feat: add set_repo_visibility in hub api (#1564) * add set_repo_visibility * fix cr * fix lint --- docker/build_image.py | 6 ++-- modelscope/hub/api.py | 79 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/docker/build_image.py b/docker/build_image.py index 4cc4075a..22fc21bd 100644 --- a/docker/build_image.py +++ b/docker/build_image.py @@ -418,10 +418,11 @@ RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy class AscendSwiftImageBuilder(SwiftImageBuilder): + def init_args(self, args) -> Any: if not args.base_image: # other vision search for: https://hub.docker.com/r/ascendai/cann/tags - args.base_image = "swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:8.3.rc1-a3-ubuntu22.04-py3.11" + args.base_image = 'swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:8.3.rc1-a3-ubuntu22.04-py3.11' return super().init_args(args) def generate_dockerfile(self) -> str: @@ -442,8 +443,7 @@ RUN pip install --no-cache-dir -U icecream soundfile pybind11 py-spy def image(self) -> str: return ( f'{docker_registry}:{self.args.base_image.split(":")[-1]}-torch2.7.1' - f'-{self.args.modelscope_version}-ascend-swift-test' - ) + f'-{self.args.modelscope_version}-ascend-swift-test') def push(self): return 0 diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 3b65e471..1a8ccc5d 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -19,7 +19,8 @@ from http import HTTPStatus from http.cookiejar import CookieJar from os.path import expanduser from pathlib import Path -from typing import Any, BinaryIO, Dict, Iterable, List, Optional, Tuple, Union +from typing import (Any, BinaryIO, Dict, Iterable, List, Literal, Optional, + Tuple, Union) from urllib.parse import urlencode import json @@ -457,7 +458,7 @@ class HubApi: model_id: str, revision: Optional[str] = DEFAULT_MODEL_REVISION, endpoint: Optional[str] = None - ) -> str: + ) -> dict: """Get model information at ModelScope Args: @@ -2868,6 +2869,80 @@ class HubApi: 'total_files': len(to_delete) } + def set_repo_visibility(self, + repo_id: str, + repo_type: Literal['model', 'dataset'], + visibility: Literal['private', 'public'], + token: Union[str, None] = None + ) -> dict: + """ + Set the visibility of a repo. + + Args: + repo_id (str): The repo id in the format of `owner_name/repo_name`. + repo_type (Literal['model', 'dataset']): The repo type, `model` or `dataset`. + visibility (Literal['private', 'public']): The visibility to set, `private` or `public`. + token (Union[str, None]): The access token. If None, will use the cookies from the local cache. + See `https://modelscope.cn/my/myaccesstoken` to get your token. + + Returns: + dict: The response from the server. + """ + if not repo_id: + raise ValueError('The arg `repo_id` cannot be empty!') + + visibility_map: Dict[str, int] = {v: k for k, v in VisibilityMap.items()} + visibility_code: int = visibility_map.get(visibility, 5) + cookies = self.get_cookies(access_token=token, cookies_required=True) + + if repo_type == REPO_TYPE_MODEL: + model_info = self.get_model(model_id=repo_id) + path = f'{self.endpoint}/api/v1/models/{repo_id}' + payload = { + 'ChineseName': model_info.get('ChineseName', ''), + 'ModelFramework': model_info.get('ModelFramework', 'Pytorch'), + 'Visibility': visibility_code, + 'ProtectedMode': 2, + 'ApprovalMode': model_info.get('ApprovalMode', 2), + 'Description': model_info.get('Description', ''), + 'AigcType': model_info.get('AigcType', ''), + 'VisionFoundation': model_info.get('VisionFoundation', ''), + 'ModelCover': model_info.get('ModelCover', ''), + 'SubScientificField': model_info.get('SubScientificField', None), + 'ScientificField': model_info.get('NEXA', {}).get('ScientificField', ''), + 'Source': model_info.get('NEXA', {}).get('Source', ''), + } + elif repo_type == REPO_TYPE_DATASET: + + repo_id_parts = repo_id.split('/') + if len(repo_id_parts) != 2 or not all(repo_id_parts): + raise ValueError(f'Invalid dataset repo_id: {repo_id}, should be in format of `owner/dataset_name`') + + dataset_idx, _ = self.get_dataset_id_and_type( + dataset_name=repo_id_parts[1], + namespace=repo_id_parts[0], + ) + + path = f'{self.endpoint}/api/v1/datasets/{dataset_idx}' + payload = { + 'Visibility': visibility_code, + 'ProtectedMode': 2, + } + else: + raise ValueError(f'Invalid repo type: {repo_type}, supported repos: {REPO_TYPE_SUPPORT}') + + r = self.session.put( + path, + json=payload, + cookies=cookies, + headers=self.builder_headers(self.headers)) + + raise_for_http_status(r) + resp = r.json() + raise_on_error(resp) + + return resp + class ModelScopeConfig: path_credential = expanduser(MODELSCOPE_CREDENTIALS_PATH) From c2891a174e2591bbd6f52488601436afda581213 Mon Sep 17 00:00:00 2001 From: "Xingjun.Wang" Date: Fri, 5 Dec 2025 15:41:22 +0800 Subject: [PATCH 14/21] Add base_model_sub_type for aigc model (#1563) --- modelscope/cli/create.py | 6 ++++++ modelscope/hub/api.py | 1 + modelscope/hub/utils/aigc.py | 4 ++++ 3 files changed, 11 insertions(+) diff --git a/modelscope/cli/create.py b/modelscope/cli/create.py index f9e9e48c..b6048532 100644 --- a/modelscope/cli/create.py +++ b/modelscope/cli/create.py @@ -129,6 +129,11 @@ class CreateCMD(CLICommand): help= 'Source of the AIGC model. `USER_UPLOAD`, `TRAINED_FROM_MODELSCOPE` or `TRAINED_FROM_ALIYUN_FC`.' ) + aigc_group.add_argument( + '--base_model_sub_type', + type=str, + default='', + help='Base model sub type, e.g., Qwen_Edit_2509') parser.set_defaults(func=subparser_func) @@ -191,6 +196,7 @@ class CreateCMD(CLICommand): base_model_id=self.args.base_model_id, path_in_repo=self.args.path_in_repo, model_source=self.args.model_source, + base_model_sub_type=self.args.base_model_sub_type, ) # Convert visibility string to int for the API call diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 1a8ccc5d..13cf53fb 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -305,6 +305,7 @@ class HubApi: 'ModelPath': aigc_model.model_path, 'TriggerWords': aigc_model.trigger_words, 'ModelSource': aigc_model.model_source, + 'SubVisionFoundation': aigc_model.base_model_sub_type, }) if aigc_model.official_tags: diff --git a/modelscope/hub/utils/aigc.py b/modelscope/hub/utils/aigc.py index 62df6cf8..2f2644ef 100644 --- a/modelscope/hub/utils/aigc.py +++ b/modelscope/hub/utils/aigc.py @@ -82,6 +82,7 @@ class AigcModel: trigger_words: Optional[List[str]] = None, official_tags: Optional[List[str]] = None, model_source: Optional[str] = 'USER_UPLOAD', + base_model_sub_type: Optional[str] = '', ): """ Initializes the AigcModel helper. @@ -99,6 +100,7 @@ class AigcModel: official_tags (List[str], optional): Official tags for the AIGC model. Defaults to None. model_source (str, optional): Source of the model. `USER_UPLOAD`, `TRAINED_FROM_MODELSCOPE` or `TRAINED_FROM_ALIYUN_FC`. Defaults to 'USER_UPLOAD'. + base_model_sub_type (str, Optional): Sub vision foundation model type. Defaults to ''. e.g. `SD_1_5` """ self.model_path = model_path self.aigc_type = aigc_type @@ -106,6 +108,7 @@ class AigcModel: self.tag = tag self.description = description self.model_source = model_source + self.base_model_sub_type = base_model_sub_type # Process cover images - convert local paths to base64 data URLs if cover_images is not None: processed_cover_images = [] @@ -391,6 +394,7 @@ class AigcModel: 'trigger_words': self.trigger_words, 'official_tags': self.official_tags, 'model_source': self.model_source, + 'base_model_sub_type': self.base_model_sub_type, } @classmethod From 12bfc59f3bcdd9970fb77e21a1ad1499771403c0 Mon Sep 17 00:00:00 2001 From: "Xingjun.Wang" Date: Mon, 8 Dec 2025 17:28:02 +0800 Subject: [PATCH 15/21] Feat: add `DEFAULT_MAX_WORKERS` in snapshot_download func (#1566) --- modelscope/hub/snapshot_download.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modelscope/hub/snapshot_download.py b/modelscope/hub/snapshot_download.py index ef9f7a01..e25daaa3 100644 --- a/modelscope/hub/snapshot_download.py +++ b/modelscope/hub/snapshot_download.py @@ -19,6 +19,7 @@ from modelscope.utils.logger import get_logger from modelscope.utils.thread_utils import thread_executor from .api import HubApi, ModelScopeConfig from .callback import ProgressCallback +from .constants import DEFAULT_MAX_WORKERS from .errors import InvalidParameter from .file_download import (create_temporary_directory_and_cache, download_file, get_file_download_url) @@ -42,7 +43,7 @@ def snapshot_download( local_dir: Optional[str] = None, allow_patterns: Optional[Union[List[str], str]] = None, ignore_patterns: Optional[Union[List[str], str]] = None, - max_workers: int = 8, + max_workers: Optional[int] = None, repo_id: str = None, repo_type: Optional[str] = REPO_TYPE_MODEL, enable_file_lock: Optional[bool] = None, @@ -112,6 +113,8 @@ def snapshot_download( f'Invalid repo type: {repo_type}, only support: {REPO_TYPE_SUPPORT}' ) + max_workers = max_workers or DEFAULT_MAX_WORKERS + if revision is None: revision = DEFAULT_DATASET_REVISION if repo_type == REPO_TYPE_DATASET else DEFAULT_MODEL_REVISION From 1213ece432f9531eb24092a0e930d2b2d05eb65c Mon Sep 17 00:00:00 2001 From: "Xingjun.Wang" Date: Mon, 8 Dec 2025 17:28:02 +0800 Subject: [PATCH 16/21] Feat: add `DEFAULT_MAX_WORKERS` in snapshot_download func (#1566) --- modelscope/hub/snapshot_download.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modelscope/hub/snapshot_download.py b/modelscope/hub/snapshot_download.py index ef9f7a01..e25daaa3 100644 --- a/modelscope/hub/snapshot_download.py +++ b/modelscope/hub/snapshot_download.py @@ -19,6 +19,7 @@ from modelscope.utils.logger import get_logger from modelscope.utils.thread_utils import thread_executor from .api import HubApi, ModelScopeConfig from .callback import ProgressCallback +from .constants import DEFAULT_MAX_WORKERS from .errors import InvalidParameter from .file_download import (create_temporary_directory_and_cache, download_file, get_file_download_url) @@ -42,7 +43,7 @@ def snapshot_download( local_dir: Optional[str] = None, allow_patterns: Optional[Union[List[str], str]] = None, ignore_patterns: Optional[Union[List[str], str]] = None, - max_workers: int = 8, + max_workers: Optional[int] = None, repo_id: str = None, repo_type: Optional[str] = REPO_TYPE_MODEL, enable_file_lock: Optional[bool] = None, @@ -112,6 +113,8 @@ def snapshot_download( f'Invalid repo type: {repo_type}, only support: {REPO_TYPE_SUPPORT}' ) + max_workers = max_workers or DEFAULT_MAX_WORKERS + if revision is None: revision = DEFAULT_DATASET_REVISION if repo_type == REPO_TYPE_DATASET else DEFAULT_MODEL_REVISION From 6d674796bac389e07fd2e3d34681b0985499c126 Mon Sep 17 00:00:00 2001 From: "Xingjun.Wang" Date: Tue, 9 Dec 2025 15:19:44 +0800 Subject: [PATCH 17/21] Fix/upload patterns (#1567) --- modelscope/hub/api.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 13cf53fb..3cebd922 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -2310,9 +2310,18 @@ class HubApi: # Ignore .git .cache folders if ignore_patterns is None: - ignore_patterns = DEFAULT_IGNORE_PATTERNS + ignore_patterns = [] elif isinstance(ignore_patterns, str): ignore_patterns = [ignore_patterns] + ignore_patterns += DEFAULT_IGNORE_PATTERNS + + # Cover the ignore patterns if both allow and ignore patterns are provided + if allow_patterns is not None: + if '**' in allow_patterns: + ignore_patterns = [] + ignore_patterns = [ + p for p in ignore_patterns if p not in allow_patterns + ] commit_message = ( commit_message if commit_message is not None else f'Upload to {repo_id} on ModelScope hub' From 62d51b9f2b8f444cefbc3bad94e315f07691dee9 Mon Sep 17 00:00:00 2001 From: suluyana <110878454+suluyana@users.noreply.github.com> Date: Tue, 9 Dec 2025 20:43:04 +0800 Subject: [PATCH 18/21] fix set_repo_visibility (#1568) Co-authored-by: suluyan --- modelscope/hub/api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 3cebd922..8118944d 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -2921,6 +2921,8 @@ class HubApi: 'SubScientificField': model_info.get('SubScientificField', None), 'ScientificField': model_info.get('NEXA', {}).get('ScientificField', ''), 'Source': model_info.get('NEXA', {}).get('Source', ''), + 'ModelTask': model_info.get('Tasks', [{}])[0].get('name'), + 'License': model_info.get('License', ''), } elif repo_type == REPO_TYPE_DATASET: From 4be91830df45834a3fecf7801f8fd14bd538bef7 Mon Sep 17 00:00:00 2001 From: suluyana <110878454+suluyana@users.noreply.github.com> Date: Tue, 9 Dec 2025 20:43:04 +0800 Subject: [PATCH 19/21] fix set_repo_visibility (#1568) Co-authored-by: suluyan --- modelscope/hub/api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 3cebd922..8118944d 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -2921,6 +2921,8 @@ class HubApi: 'SubScientificField': model_info.get('SubScientificField', None), 'ScientificField': model_info.get('NEXA', {}).get('ScientificField', ''), 'Source': model_info.get('NEXA', {}).get('Source', ''), + 'ModelTask': model_info.get('Tasks', [{}])[0].get('name'), + 'License': model_info.get('License', ''), } elif repo_type == REPO_TYPE_DATASET: From 13d8f27bf0c53d9ac7b57b519446660ed869012b Mon Sep 17 00:00:00 2001 From: "Xingjun.Wang" Date: Wed, 10 Dec 2025 10:27:20 +0800 Subject: [PATCH 20/21] add error log for invalid input visibility args (#1569) --- modelscope/hub/api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 8118944d..b2d76fd8 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -2901,6 +2901,9 @@ class HubApi: if not repo_id: raise ValueError('The arg `repo_id` cannot be empty!') + if visibility not in ['private', 'public']: + raise ValueError(f'Invalid visibility: {visibility}, supported visibilities: `private`, `public`') + visibility_map: Dict[str, int] = {v: k for k, v in VisibilityMap.items()} visibility_code: int = visibility_map.get(visibility, 5) cookies = self.get_cookies(access_token=token, cookies_required=True) From 95aaca342134ade3f3bfe15575e3df649cfed768 Mon Sep 17 00:00:00 2001 From: suluyana <110878454+suluyana@users.noreply.github.com> Date: Wed, 10 Dec 2025 11:10:57 +0800 Subject: [PATCH 21/21] fix: check tasks in HubApi.set_repo_visibility (#1570) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * bump version * Feat/update default revision (#1553) * Feat: add set_repo_visibility in hub api (#1564) * add set_repo_visibility * fix cr * fix lint * Add base_model_sub_type for aigc model (#1563) * Feat: add `DEFAULT_MAX_WORKERS` in snapshot_download func (#1566) * fix set_repo_visibility (#1568) Co-authored-by: suluyan * fix: check tasks * for merge master * fix typo --------- Co-authored-by: suluyan Co-authored-by: Gongsheng Li <58078985+alcholiclg@users.noreply.github.com> Co-authored-by: Xingjun.Wang Co-authored-by: 班扬 --- modelscope/hub/api.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index b2d76fd8..36e1fc7d 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -2911,6 +2911,12 @@ class HubApi: if repo_type == REPO_TYPE_MODEL: model_info = self.get_model(model_id=repo_id) path = f'{self.endpoint}/api/v1/models/{repo_id}' + tasks = model_info.get('Tasks') + model_tasks = '' + if isinstance(tasks, list) and tasks: + first = tasks[0] + if isinstance(first, dict) and first: + model_tasks = first.get('name') payload = { 'ChineseName': model_info.get('ChineseName', ''), 'ModelFramework': model_info.get('ModelFramework', 'Pytorch'), @@ -2924,7 +2930,7 @@ class HubApi: 'SubScientificField': model_info.get('SubScientificField', None), 'ScientificField': model_info.get('NEXA', {}).get('ScientificField', ''), 'Source': model_info.get('NEXA', {}).get('Source', ''), - 'ModelTask': model_info.get('Tasks', [{}])[0].get('name'), + 'ModelTask': model_tasks, 'License': model_info.get('License', ''), } elif repo_type == REPO_TYPE_DATASET: