From 3797b4b8d112559c10af4cc9ae82f357ac3fc699 Mon Sep 17 00:00:00 2001 From: "yaqiang.sun" Date: Wed, 13 Aug 2025 22:23:25 +0800 Subject: [PATCH 1/4] Fix intermixed missing in cli_argument_parser.py (#1437) Co-authored by: @yaqiangsun --- modelscope/trainers/cli_argument_parser.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/modelscope/trainers/cli_argument_parser.py b/modelscope/trainers/cli_argument_parser.py index 0ee1f382..37213641 100644 --- a/modelscope/trainers/cli_argument_parser.py +++ b/modelscope/trainers/cli_argument_parser.py @@ -21,12 +21,17 @@ class CliArgumentParser(ArgumentParser): def get_manual_args(self, args): return [arg[2:] for arg in args if arg.startswith('--')] - def _parse_known_args(self, args: List = None, namespace=None): + def _parse_known_args(self, + args: List = None, + namespace=None, + *args_extra, + **kwargs): self.model_id = namespace.model if namespace is not None else None if '--model' in args: self.model_id = args[args.index('--model') + 1] self.manual_args = self.get_manual_args(args) - return super()._parse_known_args(args, namespace) + return super()._parse_known_args(args, namespace, *args_extra, + **kwargs) def print_help(self, file=None): return super().print_help(file) From e8026308650b0403be5df33b55b53ad6d5612f1c Mon Sep 17 00:00:00 2001 From: Koko-ry <2024104299@ruc.edu.cn> Date: Wed, 13 Aug 2025 23:31:59 +0800 Subject: [PATCH 2/4] Fix/aigc weight (#1464) --- modelscope/hub/api.py | 4 ++- modelscope/hub/utils/aigc.py | 70 +++++++++++++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 432317f9..3e2d6f9f 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -244,6 +244,8 @@ class HubApi: if aigc_model is not None: # Use AIGC model endpoint path = f'{endpoint}/api/v1/models/aigc' + # Best-effort pre-upload weights so server recognizes sha256 (use existing cookies) + aigc_model.preupload_weights(cookies=cookies, headers=self.builder_headers(self.headers)) # Add AIGC-specific fields to body body.update({ @@ -272,7 +274,7 @@ class HubApi: raise_on_error(r.json()) model_repo_url = f'{endpoint}/models/{model_id}' - # TODO: due to server error, the upload function is not working + # TODO: to be aligned with the new api # Upload model files for AIGC models # if aigc_model is not None: # aigc_model.upload_to_repo(self, model_id, token) diff --git a/modelscope/hub/utils/aigc.py b/modelscope/hub/utils/aigc.py index 9f3dad07..0109d4ca 100644 --- a/modelscope/hub/utils/aigc.py +++ b/modelscope/hub/utils/aigc.py @@ -1,9 +1,12 @@ # Copyright (c) Alibaba, Inc. and its affiliates. - import glob import os from typing import List, Optional +import requests +from tqdm.auto import tqdm + +from modelscope.hub.utils.utils import MODELSCOPE_URL_SCHEME, get_domain from modelscope.utils.logger import get_logger logger = get_logger() @@ -209,6 +212,71 @@ class AigcModel: 'You may need to upload the model manually after creation.') return False + def preupload_weights(self, + *, + cookies: Optional[object] = None, + timeout: int = 300, + headers: Optional[dict] = None) -> None: + """Pre-upload aigc model weights to the LFS server. + + Server may require the sha256 of weights to be registered before creation. + This method streams the weight file so the sha gets registered. + + Args: + cookies: Optional requests-style cookies (CookieJar/dict). If provided, preferred. + timeout: Request timeout seconds. + headers: Optional headers. + """ + domain: str = get_domain() + base_url: str = f'{MODELSCOPE_URL_SCHEME}lfs.{domain.lstrip("www.")}' + url: str = f'{base_url}/api/v1/models/aigc/weights' + + file_path = getattr(self, 'target_file', None) or self.model_path + file_path = os.path.abspath(os.path.expanduser(file_path)) + if not os.path.isfile(file_path): + raise ValueError(f'Pre-upload expects a file, got: {file_path}') + + cookies = dict(cookies) if cookies else None + if cookies is None: + raise ValueError('Token does not exist, please login first.') + + headers.update({'Cookie': f"m_session_id={cookies['m_session_id']}"}) + + file_size = os.path.getsize(file_path) + + def read_in_chunks(file_object, + pbar, + chunk_size: int = 1 * 1024 * 1024): + while True: + ck = file_object.read(chunk_size) + if not ck: + break + pbar.update(len(ck)) + yield ck + + with tqdm( + total=file_size, + unit='B', + unit_scale=True, + dynamic_ncols=True, + desc='[Pre-uploading] ') as pbar: + with open(file_path, 'rb') as f: + r = requests.put( + url, + headers=headers, + data=read_in_chunks(f, pbar), + timeout=timeout, + ) + try: + resp = r.json() + except requests.exceptions.JSONDecodeError: + r.raise_for_status() + return + # If JSON body returned, try best-effort check + if isinstance(resp, dict) and resp.get('Success') is False: + msg = resp.get('Message', 'unknown error') + raise RuntimeError(f'Pre-upload failed: {msg}') + def to_dict(self) -> dict: """Converts the AIGC parameters to a dictionary suitable for API calls.""" return { From 7d11b77112a0e47c1a0c1a0e18b9d339e2012d13 Mon Sep 17 00:00:00 2001 From: "Xingjun.Wang" Date: Thu, 14 Aug 2025 10:56:16 +0800 Subject: [PATCH 3/4] Fix `trust_remote_code` (#1462) 1. Set `trust_remote_code` to `True` by default in datasets module 2. Set `trust_remote_code` to `True` by default in PolyLM pipeline --- .dev_scripts/ci_container_test.sh | 8 +------- modelscope/models/audio/tts/sambert_hifi.py | 3 +++ modelscope/models/nlp/polylm/text_generation.py | 13 +++++++++---- modelscope/msdatasets/ms_dataset.py | 4 ++-- modelscope/msdatasets/utils/hf_datasets_util.py | 2 +- tests/trainers/audio/test_tts_trainer.py | 4 +++- 6 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.dev_scripts/ci_container_test.sh b/.dev_scripts/ci_container_test.sh index 45eac84b..a66a3cd3 100644 --- a/.dev_scripts/ci_container_test.sh +++ b/.dev_scripts/ci_container_test.sh @@ -21,13 +21,7 @@ if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then fi fi - pip install -r requirements/framework.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html - pip install -r requirements/audio.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html - pip install -r requirements/cv.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html - pip install -r requirements/multi-modal.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html - pip install -r requirements/nlp.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html - pip install -r requirements/science.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html - + pip install -U sentence_transformers python -m spacy download en_core_web_sm pip install faiss-gpu pip install healpy diff --git a/modelscope/models/audio/tts/sambert_hifi.py b/modelscope/models/audio/tts/sambert_hifi.py index 6df9ec97..a9896eac 100644 --- a/modelscope/models/audio/tts/sambert_hifi.py +++ b/modelscope/models/audio/tts/sambert_hifi.py @@ -5,6 +5,7 @@ from __future__ import (absolute_import, division, print_function, import datetime import os import shutil +import sys import wave import zipfile @@ -60,6 +61,8 @@ class SambertHifigan(Model): raise TtsVoiceNotExistsException( 'modelscope error: voices is empty in voices.json') # initialize frontend + if sys.version_info >= (3, 11): + raise ImportError('Python version needs to be <= 3.10') import ttsfrd frontend = ttsfrd.TtsFrontendEngine() zip_file = os.path.join(model_dir, 'resource.zip') diff --git a/modelscope/models/nlp/polylm/text_generation.py b/modelscope/models/nlp/polylm/text_generation.py index bd6fbd69..5b7761a8 100644 --- a/modelscope/models/nlp/polylm/text_generation.py +++ b/modelscope/models/nlp/polylm/text_generation.py @@ -9,7 +9,6 @@ from modelscope.metainfo import Models from modelscope.models.base import Tensor, TorchModel from modelscope.models.builder import MODELS from modelscope.utils.constant import Tasks -from modelscope.utils.hub import read_config from modelscope.utils.logger import get_logger from modelscope.utils.streaming_output import StreamingOutputMixin @@ -30,11 +29,17 @@ class PolyLMForTextGeneration(TorchModel, StreamingOutputMixin): super().__init__(model_dir, *args, **kwargs) self.tokenizer = AutoTokenizer.from_pretrained( model_dir, legacy=False, use_fast=False) - logger.warning( + + self.check_trust_remote_code( + info_str= f'Use trust_remote_code=True. Will invoke codes from {model_dir}. Please make sure ' - 'that you can trust the external codes.') + 'that you can trust the external codes.', + model_dir=model_dir) + self.model = AutoModelForCausalLM.from_pretrained( - model_dir, device_map='auto', trust_remote_code=True) + model_dir, + device_map='auto', + trust_remote_code=self.trust_remote_code) self.model.eval() def forward(self, input: Dict[str, Tensor], **kwargs) -> Dict[str, Tensor]: diff --git a/modelscope/msdatasets/ms_dataset.py b/modelscope/msdatasets/ms_dataset.py index cc67b315..81f65652 100644 --- a/modelscope/msdatasets/ms_dataset.py +++ b/modelscope/msdatasets/ms_dataset.py @@ -171,7 +171,7 @@ class MsDataset: custom_cfg: Optional[Config] = Config(), token: Optional[str] = None, dataset_info_only: Optional[bool] = False, - trust_remote_code: Optional[bool] = True, + trust_remote_code: Optional[bool] = False, **config_kwargs, ) -> Union[dict, 'MsDataset', NativeIterableDataset]: """Load a MsDataset from the ModelScope Hub, Hugging Face Hub, urls, or a local dataset. @@ -202,7 +202,7 @@ class MsDataset: see https://modelscope.cn/docs/Configuration%E8%AF%A6%E8%A7%A3 token (str, Optional): SDK token of ModelScope. dataset_info_only (bool, Optional): If set to True, only return the dataset config and info (dict). - trust_remote_code (bool, Optional): If set to True, trust the remote code. + trust_remote_code (bool, Optional): If set to True, trust the remote code. Default to `False`. **config_kwargs (additional keyword arguments): Keyword arguments to be passed Returns: diff --git a/modelscope/msdatasets/utils/hf_datasets_util.py b/modelscope/msdatasets/utils/hf_datasets_util.py index 9053d062..f867081e 100644 --- a/modelscope/msdatasets/utils/hf_datasets_util.py +++ b/modelscope/msdatasets/utils/hf_datasets_util.py @@ -940,7 +940,7 @@ class DatasetsWrapperHF: streaming: bool = False, num_proc: Optional[int] = None, storage_options: Optional[Dict] = None, - trust_remote_code: bool = True, + trust_remote_code: bool = False, dataset_info_only: Optional[bool] = False, **config_kwargs, ) -> Union[DatasetDict, Dataset, IterableDatasetDict, IterableDataset, diff --git a/tests/trainers/audio/test_tts_trainer.py b/tests/trainers/audio/test_tts_trainer.py index 3792729d..117a78a9 100644 --- a/tests/trainers/audio/test_tts_trainer.py +++ b/tests/trainers/audio/test_tts_trainer.py @@ -45,7 +45,9 @@ class TestTtsTrainer(unittest.TestCase): shutil.rmtree(self.tmp_dir, ignore_errors=True) super().tearDown() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless( + test_level() >= 2, + 'skip test because the ci test python version is higher then 3.10') def test_trainer(self): kwargs = dict( model=self.model_id, From 7bb94a3cf1dc864a0bf290e53d3917aef2781741 Mon Sep 17 00:00:00 2001 From: Koko-ry <2024104299@ruc.edu.cn> Date: Thu, 14 Aug 2025 16:06:57 +0800 Subject: [PATCH 4/4] Fix: filter existing files (.gitattributes/configuration.json/README.md) --- modelscope/hub/utils/aigc.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/modelscope/hub/utils/aigc.py b/modelscope/hub/utils/aigc.py index 0109d4ca..7be6a7c9 100644 --- a/modelscope/hub/utils/aigc.py +++ b/modelscope/hub/utils/aigc.py @@ -75,6 +75,8 @@ class AigcModel: cover_images (List[str], optional): List of cover image URLs. base_model_id (str, optional): Base model name. e.g., 'AI-ModelScope/FLUX.1-dev'. path_in_repo (str, optional): Path in the repository. + Note: Auto-upload during AIGC create is temporarily disabled by server. This parameter + will not take effect at creation time. """ self.model_path = model_path self.aigc_type = aigc_type @@ -125,6 +127,30 @@ class AigcModel: target_file = self.model_path logger.info('Using file: %s', os.path.basename(target_file)) elif os.path.isdir(self.model_path): + # Validate top-level directory: it must not be empty; and if it has files, + # they must not be only the common placeholder files + top_entries = os.listdir(self.model_path) + if len(top_entries) == 0: + raise ValueError( + f'Directory is empty: {self.model_path}. ' + f'Please place at least one model file at the top level (e.g., .safetensors/.pth/.bin).' + ) + + top_files = [ + name for name in top_entries + if os.path.isfile(os.path.join(self.model_path, name)) + ] + placeholder_names = { + '.gitattributes', 'configuration.json', 'readme.md' + } + if top_files: + normalized = {name.lower() for name in top_files} + if normalized.issubset(placeholder_names): + raise ValueError( + 'Top-level directory contains only [.gitattributes, configuration.json, README.md]. ' + 'Please place additional model files at the top level (e.g., .safetensors/.pth/.bin).' + ) + # Priority order for metadata file: safetensors -> pth -> bin -> first file file_extensions = ['.safetensors', '.pth', '.bin'] target_file = None