From a3b2c7ae31d2e2ad647f1d000b8ca35875d0f9a4 Mon Sep 17 00:00:00 2001 From: suluyana Date: Thu, 2 Jan 2025 17:24:10 +0800 Subject: [PATCH 01/13] bump version 1.22.0 --- modelscope/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modelscope/version.py b/modelscope/version.py index 031a86b4..2ace39be 100644 --- a/modelscope/version.py +++ b/modelscope/version.py @@ -1,5 +1,5 @@ # Make sure to modify __release_datetime__ to release time when making official release. -__version__ = '2.0.0' +__version__ = '1.22.0' # default release datetime for branches under active development is set # to be a time far-far-away-into-the-future -__release_datetime__ = '2099-09-06 00:00:00' +__release_datetime__ = '2025-01-02 23:59:59' From 776398bcab36a93aaf2125d6dff5e34ab610faff Mon Sep 17 00:00:00 2001 From: suluyana Date: Thu, 2 Jan 2025 17:44:28 +0800 Subject: [PATCH 02/13] fix confict between lmdeploy & vllm --- docker/build_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/build_image.py b/docker/build_image.py index 344fc9d3..8ba4fd68 100644 --- a/docker/build_image.py +++ b/docker/build_image.py @@ -284,7 +284,7 @@ class LLMImageBuilder(Builder): self.args.python_version) version_args = ( f'{self.args.torch_version} {self.args.torchvision_version} {self.args.torchaudio_version} ' - f'{self.args.vllm_version} {self.args.lmdeploy_version} {self.args.autogptq_version}' + f'{self.args.lmdeploy_version} {self.args.vllm_version} {self.args.autogptq_version}' ) with open('docker/Dockerfile.ubuntu', 'r') as f: content = f.read() From 24876ae74c902cec3bb5245d76de8d9f5d02e424 Mon Sep 17 00:00:00 2001 From: suluyana Date: Mon, 6 Jan 2025 14:36:16 +0800 Subject: [PATCH 03/13] flash-attn version --- docker/build_image.py | 12 +++++++++--- docker/install.sh | 4 +++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docker/build_image.py b/docker/build_image.py index 8ba4fd68..72067a57 100644 --- a/docker/build_image.py +++ b/docker/build_image.py @@ -36,6 +36,8 @@ class Builder: args.lmdeploy_version = '0.6.2' if not args.autogptq_version: args.autogptq_version = '0.7.1' + if not args.flashattn_version: + args.flashattn_version = '2.7.1.post4' return args def _generate_cudatoolkit_version(self, cuda_version: str) -> str: @@ -209,7 +211,8 @@ RUN pip install tf-keras==2.16.0 --no-dependencies && \ version_args = ( f'{self.args.torch_version} {self.args.torchvision_version} {self.args.torchaudio_version} ' - f'{self.args.vllm_version} {self.args.lmdeploy_version} {self.args.autogptq_version}' + f'{self.args.vllm_version} {self.args.lmdeploy_version} {self.args.autogptq_version} ' + f'{self.args.flashattn_version}' ) base_image = ( f'{docker_registry}:ubuntu{self.args.ubuntu_version}-cuda{self.args.cuda_version}-{self.args.python_tag}-' @@ -274,6 +277,8 @@ class LLMImageBuilder(Builder): args.lmdeploy_version = '0.6.2' if not args.autogptq_version: args.autogptq_version = '0.7.1' + if not args.flashattn_version: + args.flashattn_version = '2.7.1.post4' return args def generate_dockerfile(self) -> str: @@ -284,7 +289,8 @@ class LLMImageBuilder(Builder): self.args.python_version) version_args = ( f'{self.args.torch_version} {self.args.torchvision_version} {self.args.torchaudio_version} ' - f'{self.args.lmdeploy_version} {self.args.vllm_version} {self.args.autogptq_version}' + f'{self.args.lmdeploy_version} {self.args.vllm_version} {self.args.autogptq_version} ' + f'{self.args.flashattn_version}' ) with open('docker/Dockerfile.ubuntu', 'r') as f: content = f.read() @@ -341,12 +347,12 @@ parser.add_argument('--torchaudio_version', type=str, default=None) parser.add_argument('--tf_version', type=str, default=None) parser.add_argument('--vllm_version', type=str, default=None) parser.add_argument('--lmdeploy_version', type=str, default=None) +parser.add_argument('--flashattn_version', type=str, default=None) parser.add_argument('--autogptq_version', type=str, default=None) parser.add_argument('--modelscope_branch', type=str, default='master') parser.add_argument('--modelscope_version', type=str, default='9.99.0') parser.add_argument('--swift_branch', type=str, default='main') parser.add_argument('--dry_run', type=int, default=0) - args = parser.parse_args() if args.image_type.lower() == 'base_cpu': diff --git a/docker/install.sh b/docker/install.sh index 3a6ffc13..d7d367dc 100644 --- a/docker/install.sh +++ b/docker/install.sh @@ -6,6 +6,7 @@ torchaudio_version=${3:-2.4.0} vllm_version=${4:-0.6.0} lmdeploy_version=${5:-0.6.1} autogptq_version=${6:-0.7.1} +flashattn_version=${7:-2.7.1.post4} pip install --no-cache-dir -U autoawq lmdeploy==$lmdeploy_version @@ -17,7 +18,8 @@ pip install --no-cache-dir tiktoken transformers_stream_generator bitsandbytes d # pip install https://github.com/Dao-AILab/flash-attention/releases/download/v2.6.3/flash_attn-2.6.3+cu123torch2.4cxx11abiTRUE-cp310-cp310-linux_x86_64.whl # find on: https://github.com/Dao-AILab/flash-attention/releases -cd /tmp && git clone https://github.com/Dao-AILab/flash-attention.git && cd flash-attention && python setup.py install && cd / && rm -fr /tmp/flash-attention && pip cache purge; +# cd /tmp && git clone https://github.com/Dao-AILab/flash-attention.git && cd flash-attention && python setup.py install && cd / && rm -fr /tmp/flash-attention && pip cache purge; +pip install --no-cache-dir flash_attn==$flashattn_version pip install --no-cache-dir triton auto-gptq==$autogptq_version vllm==$vllm_version -U && pip cache purge From 49d80d9c1d34b158dd1f41d4ff573063e1512087 Mon Sep 17 00:00:00 2001 From: suluyana Date: Mon, 6 Jan 2025 21:40:53 +0800 Subject: [PATCH 04/13] fix version building image --- docker/build_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/build_image.py b/docker/build_image.py index 72067a57..b06858ec 100644 --- a/docker/build_image.py +++ b/docker/build_image.py @@ -289,7 +289,7 @@ class LLMImageBuilder(Builder): self.args.python_version) version_args = ( f'{self.args.torch_version} {self.args.torchvision_version} {self.args.torchaudio_version} ' - f'{self.args.lmdeploy_version} {self.args.vllm_version} {self.args.autogptq_version} ' + f'{self.args.vllm_version} {self.args.lmdeploy_version} {self.args.autogptq_version} ' f'{self.args.flashattn_version}' ) with open('docker/Dockerfile.ubuntu', 'r') as f: From b5b08f9718100d0a68efbf29d203a3611979312e Mon Sep 17 00:00:00 2001 From: tastelikefeet <58414341+tastelikefeet@users.noreply.github.com> Date: Mon, 6 Jan 2025 17:29:44 +0800 Subject: [PATCH 05/13] fix https://www.modelscope.cn/models/iic/nlp_structbert_address-parsing_chinese_base/feedback/issueDetail/20431 (#1170) --- .../preprocessors/nlp/token_classification_preprocessor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modelscope/preprocessors/nlp/token_classification_preprocessor.py b/modelscope/preprocessors/nlp/token_classification_preprocessor.py index b3ff9935..902dafca 100644 --- a/modelscope/preprocessors/nlp/token_classification_preprocessor.py +++ b/modelscope/preprocessors/nlp/token_classification_preprocessor.py @@ -416,6 +416,8 @@ class TokenClassificationTransformersPreprocessor( offset_mapping = [] tokens = self.nlp_tokenizer.tokenizer.tokenize(text) offset = 0 + if getattr(self.nlp_tokenizer.tokenizer, 'do_lower_case', False): + text = text.lower() for token in tokens: is_start = (token[:2] != '##') if is_start: From b2cdb723df6afc29d44863ac8cc569e0a9e5c869 Mon Sep 17 00:00:00 2001 From: Yingda Chen Date: Fri, 10 Jan 2025 13:48:44 +0800 Subject: [PATCH 06/13] fix path contatenation to be windows compatabile (#1176) * fix path contatenation to be windows compatabile * support dataset too --------- Co-authored-by: Yingda Chen --- modelscope/hub/snapshot_download.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modelscope/hub/snapshot_download.py b/modelscope/hub/snapshot_download.py index 4510280b..2f7f4790 100644 --- a/modelscope/hub/snapshot_download.py +++ b/modelscope/hub/snapshot_download.py @@ -233,7 +233,7 @@ def _snapshot_download( if repo_type == REPO_TYPE_MODEL: directory = os.path.abspath( local_dir) if local_dir is not None else os.path.join( - system_cache, repo_id) + system_cache, *repo_id.split('/')) print(f'Downloading Model to directory: {directory}') revision_detail = _api.get_valid_revision_detail( repo_id, revision=revision, cookies=cookies) @@ -294,7 +294,7 @@ def _snapshot_download( elif repo_type == REPO_TYPE_DATASET: directory = os.path.abspath( local_dir) if local_dir else os.path.join( - system_cache, 'datasets', repo_id) + system_cache, 'datasets', *repo_id.split('/')) print(f'Downloading Dataset to directory: {directory}') group_or_owner, name = model_id_to_group_owner_name(repo_id) From facf8dab12984a037627bd272b88c532aee23f38 Mon Sep 17 00:00:00 2001 From: suluyana <110878454+suluyana@users.noreply.github.com> Date: Fri, 10 Jan 2025 15:38:05 +0800 Subject: [PATCH 07/13] logger.warning when using remote code (#1171) * logger warning when using remote code Co-authored-by: suluyan --- modelscope/models/nlp/polylm/text_generation.py | 6 ++++++ modelscope/msdatasets/data_loader/data_loader.py | 5 +++++ .../data_loader/data_loader_manager.py | 9 +++++++++ modelscope/msdatasets/ms_dataset.py | 5 +++++ modelscope/msdatasets/utils/hf_datasets_util.py | 16 ++++++++++++++++ modelscope/pipelines/accelerate/vllm.py | 6 ++++++ .../pipelines/multi_modal/ovis_vl_pipeline.py | 7 ++++++- modelscope/pipelines/nlp/llm_pipeline.py | 12 ++++++++++++ .../pipelines/nlp/text_generation_pipeline.py | 12 ++++++++++++ modelscope/preprocessors/templates/loader.py | 3 +++ modelscope/utils/automodel_utils.py | 6 ++++++ modelscope/utils/plugins.py | 3 +++ 12 files changed, 89 insertions(+), 1 deletion(-) diff --git a/modelscope/models/nlp/polylm/text_generation.py b/modelscope/models/nlp/polylm/text_generation.py index 1881cf2b..bd6fbd69 100644 --- a/modelscope/models/nlp/polylm/text_generation.py +++ b/modelscope/models/nlp/polylm/text_generation.py @@ -10,8 +10,11 @@ from modelscope.models.base import Tensor, TorchModel from modelscope.models.builder import MODELS from modelscope.utils.constant import Tasks from modelscope.utils.hub import read_config +from modelscope.utils.logger import get_logger from modelscope.utils.streaming_output import StreamingOutputMixin +logger = get_logger() + __all__ = ['PolyLMForTextGeneration'] @@ -27,6 +30,9 @@ class PolyLMForTextGeneration(TorchModel, StreamingOutputMixin): super().__init__(model_dir, *args, **kwargs) self.tokenizer = AutoTokenizer.from_pretrained( model_dir, legacy=False, use_fast=False) + logger.warning( + f'Use trust_remote_code=True. Will invoke codes from {model_dir}. Please make sure ' + 'that you can trust the external codes.') self.model = AutoModelForCausalLM.from_pretrained( model_dir, device_map='auto', trust_remote_code=True) self.model.eval() diff --git a/modelscope/msdatasets/data_loader/data_loader.py b/modelscope/msdatasets/data_loader/data_loader.py index 92074449..71fcee45 100644 --- a/modelscope/msdatasets/data_loader/data_loader.py +++ b/modelscope/msdatasets/data_loader/data_loader.py @@ -133,6 +133,11 @@ class OssDownloader(BaseDownloader): raise f'meta-file: {dataset_name}.py not found on the modelscope hub.' if dataset_py_script and dataset_formation == DatasetFormations.hf_compatible: + if trust_remote_code: + logger.warning( + f'Use trust_remote_code=True. Will invoke codes from {dataset_name}. Please make ' + 'sure that you can trust the external codes.') + self.dataset = hf_load_dataset( dataset_py_script, name=subset_name, diff --git a/modelscope/msdatasets/data_loader/data_loader_manager.py b/modelscope/msdatasets/data_loader/data_loader_manager.py index a9e58b7c..b64a8926 100644 --- a/modelscope/msdatasets/data_loader/data_loader_manager.py +++ b/modelscope/msdatasets/data_loader/data_loader_manager.py @@ -71,6 +71,11 @@ class LocalDataLoaderManager(DataLoaderManager): # Select local data loader # TODO: more loaders to be supported. if data_loader_type == LocalDataLoaderType.HF_DATA_LOADER: + if trust_remote_code: + logger.warning( + f'Use trust_remote_code=True. Will invoke codes from {dataset_name}. Please make ' + 'sure that you can trust the external codes.') + # Build huggingface data loader and return dataset. return hf_data_loader( dataset_name, @@ -110,6 +115,10 @@ class RemoteDataLoaderManager(DataLoaderManager): # To use the huggingface data loader if data_loader_type == RemoteDataLoaderType.HF_DATA_LOADER: + if trust_remote_code: + logger.warning( + f'Use trust_remote_code=True. Will invoke codes from {dataset_name}. Please make ' + 'sure that you can trust the external codes.') dataset_ret = hf_data_loader( dataset_name, name=subset_name, diff --git a/modelscope/msdatasets/ms_dataset.py b/modelscope/msdatasets/ms_dataset.py index 899142ad..dbe15171 100644 --- a/modelscope/msdatasets/ms_dataset.py +++ b/modelscope/msdatasets/ms_dataset.py @@ -237,6 +237,11 @@ class MsDataset: if not namespace or not dataset_name: raise 'The dataset_name should be in the form of `namespace/dataset_name` or `dataset_name`.' + if trust_remote_code: + logger.warning( + f'Use trust_remote_code=True. Will invoke codes from {dataset_name}. Please make sure that ' + 'you can trust the external codes.') + # Init context config dataset_context_config = DatasetContextConfig( dataset_name=dataset_name, diff --git a/modelscope/msdatasets/utils/hf_datasets_util.py b/modelscope/msdatasets/utils/hf_datasets_util.py index 4d6de81c..fea304f6 100644 --- a/modelscope/msdatasets/utils/hf_datasets_util.py +++ b/modelscope/msdatasets/utils/hf_datasets_util.py @@ -835,6 +835,8 @@ def get_module_with_script(self) -> DatasetModule: if not os.path.exists(importable_file_path): trust_remote_code = resolve_trust_remote_code(trust_remote_code=self.trust_remote_code, repo_id=self.name) if trust_remote_code: + logger.warning(f'Use trust_remote_code=True. Will invoke codes from {repo_id}. Please make sure that ' + 'you can trust the external codes.') _create_importable_file( local_path=local_script_path, local_imports=local_imports, @@ -934,6 +936,11 @@ class DatasetsWrapperHF: verification_mode or VerificationMode.BASIC_CHECKS ) if not save_infos else VerificationMode.ALL_CHECKS) + if trust_remote_code: + logger.warning(f'Use trust_remote_code=True. Will invoke codes from {path}. Please make sure ' + 'that you can trust the external codes.' + ) + # Create a dataset builder builder_instance = DatasetsWrapperHF.load_dataset_builder( path=path, @@ -1061,6 +1068,11 @@ class DatasetsWrapperHF: ) if download_config else DownloadConfig() download_config.storage_options.update(storage_options) + if trust_remote_code: + logger.warning(f'Use trust_remote_code=True. Will invoke codes from {path}. Please make sure ' + 'that you can trust the external codes.' + ) + dataset_module = DatasetsWrapperHF.dataset_module_factory( path, revision=revision, @@ -1171,6 +1183,10 @@ class DatasetsWrapperHF: # -> the module from the python file in the dataset repository # - if path has one "/" and is dataset repository on the HF hub without a python file # -> use a packaged module (csv, text etc.) based on content of the repository + if trust_remote_code: + logger.warning(f'Use trust_remote_code=True. Will invoke codes from {path}. Please make sure ' + 'that you can trust the external codes.' + ) # Try packaged if path in _PACKAGED_DATASETS_MODULES: diff --git a/modelscope/pipelines/accelerate/vllm.py b/modelscope/pipelines/accelerate/vllm.py index 15ced4bb..9cadb979 100644 --- a/modelscope/pipelines/accelerate/vllm.py +++ b/modelscope/pipelines/accelerate/vllm.py @@ -1,8 +1,11 @@ from typing import List, Union +from modelscope import get_logger from modelscope.pipelines.accelerate.base import InferFramework from modelscope.utils.import_utils import is_vllm_available +logger = get_logger() + class Vllm(InferFramework): @@ -27,6 +30,9 @@ class Vllm(InferFramework): if not Vllm.check_gpu_compatibility(8) and (dtype in ('bfloat16', 'auto')): dtype = 'float16' + logger.warning( + f'Use trust_remote_code=True. Will invoke codes from {self.model_dir}. Please make ' + 'sure that you can trust the external codes.') self.model = LLM( self.model_dir, dtype=dtype, diff --git a/modelscope/pipelines/multi_modal/ovis_vl_pipeline.py b/modelscope/pipelines/multi_modal/ovis_vl_pipeline.py index f19eddff..cdce09a1 100644 --- a/modelscope/pipelines/multi_modal/ovis_vl_pipeline.py +++ b/modelscope/pipelines/multi_modal/ovis_vl_pipeline.py @@ -2,7 +2,7 @@ from typing import Any, Dict, Union import torch -from modelscope import AutoModelForCausalLM +from modelscope import AutoModelForCausalLM, get_logger from modelscope.metainfo import Pipelines, Preprocessors from modelscope.models.base import Model from modelscope.outputs import OutputKeys @@ -13,6 +13,8 @@ from modelscope.pipelines.multi_modal.visual_question_answering_pipeline import from modelscope.preprocessors import Preprocessor, load_image from modelscope.utils.constant import Fields, Frameworks, Tasks +logger = get_logger() + @PIPELINES.register_module( Tasks.visual_question_answering, module_name='ovis-vl') @@ -35,6 +37,9 @@ class VisionChatPipeline(VisualQuestionAnsweringPipeline): torch_dtype = kwargs.get('torch_dtype', torch.float16) multimodal_max_length = kwargs.get('multimodal_max_length', 8192) self.device = 'cuda' if device == 'gpu' else device + logger.warning( + f'Use trust_remote_code=True. Will invoke codes from {model}. Please make ' + 'sure that you can trust the external codes.') self.model = AutoModelForCausalLM.from_pretrained( model, torch_dtype=torch_dtype, diff --git a/modelscope/pipelines/nlp/llm_pipeline.py b/modelscope/pipelines/nlp/llm_pipeline.py index 7ad0d278..269e8a42 100644 --- a/modelscope/pipelines/nlp/llm_pipeline.py +++ b/modelscope/pipelines/nlp/llm_pipeline.py @@ -97,6 +97,9 @@ class LLMPipeline(Pipeline, PipelineStreamingOutputMixin): assert base_model is not None, 'Cannot get adapter_cfg.model_id_or_path from configuration.json file.' revision = self.cfg.safe_get('adapter_cfg.model_revision', 'master') + logger.warning( + f'Use trust_remote_code=True. Will invoke codes from {base_model}. Please make sure that you can ' + 'trust the external codes.') base_model = Model.from_pretrained( base_model, revision, @@ -134,6 +137,9 @@ class LLMPipeline(Pipeline, PipelineStreamingOutputMixin): model) else snapshot_download(model) # TODO: Temporary use of AutoModelForCausalLM # Need to be updated into a universal solution + logger.warning( + f'Use trust_remote_code=True. Will invoke codes from {model_dir}. Please make sure ' + 'that you can trust the external codes.') model = AutoModelForCausalLM.from_pretrained( model_dir, device_map=self.device_map, @@ -173,6 +179,9 @@ class LLMPipeline(Pipeline, PipelineStreamingOutputMixin): self.llm_framework = llm_framework if os.path.exists(kwargs['model']): + logger.warning( + f'Use trust_remote_code=True. Will invoke codes from {kwargs["model"]}. Please make sure ' + 'that you can trust the external codes.') config = AutoConfig.from_pretrained( kwargs['model'], trust_remote_code=True) q_config = config.__dict__.get('quantization_config', None) @@ -423,6 +432,9 @@ class LLMPipeline(Pipeline, PipelineStreamingOutputMixin): model_dir = self.model.model_dir if tokenizer_class is None: tokenizer_class = AutoTokenizer + logger.warning( + f'Use trust_remote_code=True. Will invoke codes from {model_dir}. Please make sure ' + 'that you can trust the external codes.') return tokenizer_class.from_pretrained( model_dir, trust_remote_code=True) diff --git a/modelscope/pipelines/nlp/text_generation_pipeline.py b/modelscope/pipelines/nlp/text_generation_pipeline.py index 55eaf809..374381cf 100644 --- a/modelscope/pipelines/nlp/text_generation_pipeline.py +++ b/modelscope/pipelines/nlp/text_generation_pipeline.py @@ -269,6 +269,9 @@ class ChatGLM6bV2TextGenerationPipeline(Pipeline): if use_bf16: default_torch_dtype = torch.bfloat16 torch_dtype = kwargs.get('torch_dtype', default_torch_dtype) + logger.warning( + f'Use trust_remote_code=True. Will invoke codes from {model_dir}. Please make sure ' + 'that you can trust the external codes.') model = Model.from_pretrained( model_dir, trust_remote_code=True, @@ -285,6 +288,9 @@ class ChatGLM6bV2TextGenerationPipeline(Pipeline): self.model = model self.model.eval() + logger.warning( + f'Use trust_remote_code=True. Will invoke codes from {self.model.model_dir}. Please ' + 'make sure that you can trust the external codes.') self.tokenizer = AutoTokenizer.from_pretrained( self.model.model_dir, trust_remote_code=True) @@ -328,6 +334,9 @@ class QWenChatPipeline(Pipeline): bf16 = False if isinstance(model, str): + logger.warning( + f'Use trust_remote_code=True. Will invoke codes from {model}. Please make sure ' + 'that you can trust the external codes.') self.tokenizer = AutoTokenizer.from_pretrained( model, revision=revision, trust_remote_code=True) self.model = AutoModelForCausalLM.from_pretrained( @@ -392,6 +401,9 @@ class QWenTextGenerationPipeline(Pipeline): bf16 = False if isinstance(model, str): + logger.warning( + f'Use trust_remote_code=True. Will invoke codes from {model}. Please make sure ' + 'that you can trust the external codes.') self.model = AutoModelForCausalLM.from_pretrained( model, device_map=device_map, diff --git a/modelscope/preprocessors/templates/loader.py b/modelscope/preprocessors/templates/loader.py index 8943f25d..8dac8d41 100644 --- a/modelscope/preprocessors/templates/loader.py +++ b/modelscope/preprocessors/templates/loader.py @@ -820,6 +820,9 @@ class TemplateLoader: model_id, revision=kwargs.pop('revision', 'master'), ignore_file_pattern=ignore_file_pattern) + logger.warning(f'Use trust_remote_code=True. Will invoke codes from {model_dir}.' + ' Please make sure that you can trust the external codes.' + ) tokenizer = AutoTokenizer.from_pretrained( model_dir, trust_remote_code=True) config = AutoConfig.from_pretrained(model_dir, trust_remote_code=True) diff --git a/modelscope/utils/automodel_utils.py b/modelscope/utils/automodel_utils.py index eb4aa6c8..9bf1459e 100644 --- a/modelscope/utils/automodel_utils.py +++ b/modelscope/utils/automodel_utils.py @@ -3,12 +3,15 @@ import os from types import MethodType from typing import Any, Optional +from modelscope import get_logger from modelscope.metainfo import Tasks from modelscope.utils.ast_utils import INDEX_KEY from modelscope.utils.import_utils import (LazyImportModule, is_torch_available, is_transformers_available) +logger = get_logger() + def can_load_by_ms(model_dir: str, task_name: Optional[str], model_type: Optional[str]) -> bool: @@ -91,6 +94,9 @@ def get_hf_automodel_class(model_dir: str, if not os.path.exists(config_path): return None try: + logger.warning( + f'Use trust_remote_code=True. Will invoke codes from {model_dir}. Please make sure ' + 'that you can trust the external codes.') config = AutoConfig.from_pretrained(model_dir, trust_remote_code=True) if task_name is None: automodel_class = get_default_automodel(config) diff --git a/modelscope/utils/plugins.py b/modelscope/utils/plugins.py index 1f191a8d..671b1ea6 100644 --- a/modelscope/utils/plugins.py +++ b/modelscope/utils/plugins.py @@ -451,6 +451,9 @@ def register_plugins_repo(plugins: List[str]) -> None: def register_modelhub_repo(model_dir, allow_remote=False) -> None: """ Try to install and import remote model from modelhub""" if allow_remote: + logger.warning( + f'Use allow_remote=True. Will invoke codes from {model_dir}. Please make sure ' + 'that you can trust the external codes.') try: import_module_from_model_dir(model_dir) except KeyError: From 9441eb731030baeacc17fea59371c05285f8c00b Mon Sep 17 00:00:00 2001 From: suluyana <110878454+suluyana@users.noreply.github.com> Date: Fri, 10 Jan 2025 15:42:22 +0800 Subject: [PATCH 08/13] feat: all other ollama models (#1174) * add cases * new models --------- Co-authored-by: suluyan --- modelscope/preprocessors/templates/loader.py | 120 +++++++++++++++---- tests/tools/test_to_ollama.py | 54 +++++++++ 2 files changed, 153 insertions(+), 21 deletions(-) diff --git a/modelscope/preprocessors/templates/loader.py b/modelscope/preprocessors/templates/loader.py index 8dac8d41..ae7460d1 100644 --- a/modelscope/preprocessors/templates/loader.py +++ b/modelscope/preprocessors/templates/loader.py @@ -230,6 +230,10 @@ template_info = [ modelfile_prefix= 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/dolphin-mistral', ), + TemplateInfo( + template_regex=f'.*{cases("dolphin3", "dolphin-3")}.*', + modelfile_prefix= + 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/dolphin3'), # "phi" TemplateInfo( @@ -251,6 +255,12 @@ template_info = [ modelfile_prefix= 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/phi3', ), + TemplateInfo( + template_regex= + f'.*{cases("phi4", "phi-4")}{no_multi_modal()}.*', + modelfile_prefix= + 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/phi4', + ), TemplateInfo( template_regex= f'.*{cases("phi")}{no_multi_modal()}.*', @@ -591,7 +601,7 @@ template_info = [ template_regex= f'.*{cases("deepseek")}.*{cases("v2")}{no("v2.5")}{no_multi_modal()}.*{chat_suffix}.*', modelfile_prefix= - 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/deepseek_v2', + 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/deepseek-v2', ), # deepseek_coder @@ -623,6 +633,94 @@ template_info = [ template=TemplateType.telechat_v2, template_regex=f'.*{cases("TeleChat")}.*{cases("v2")}.*'), + # tulu3 + TemplateInfo( + template_regex=f'.*{cases("tulu3", "tulu-3")}.*', + modelfile_prefix= + 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/tulu3'), + + # athene-v2 + TemplateInfo( + template_regex=f'.*{cases("athene-v2")}.*', + modelfile_prefix= + 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/athene-v2'), + + # granite + TemplateInfo( + template_regex=f'.*{cases("granite-guardian-3")}.*', + modelfile_prefix= + 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/granite3-guardian'), + TemplateInfo( + template_regex=f'.*{cases("granite")}.*{cases("code")}.*', + modelfile_prefix= + 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/granite-code'), + TemplateInfo( + template_regex=f'.*{cases("granite-3.1")}.*{cases("2b", "8b")}.*', + modelfile_prefix= + 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/granite3.1-dense'), + TemplateInfo( + template_regex=f'.*{cases("granite-3.1")}.*{cases("1b", "3b")}.*', + modelfile_prefix= + 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/granite3.1-moe'), + TemplateInfo( + template_regex=f'.*{cases("granite-embedding")}.*', + modelfile_prefix= + 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/granite-embedding'), + TemplateInfo( + template_regex=f'.*{cases("granite-3")}.*{cases("2b", "8b")}.*', + modelfile_prefix= + 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/granite3-dense'), + TemplateInfo( + template_regex=f'.*{cases("granite-3")}.*{cases("1b", "3b")}.*', + modelfile_prefix= + 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/granite3-moe'), + + # opencoder + TemplateInfo( + template_regex=f'.*{cases("opencoder")}.*', + modelfile_prefix= + 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/opencoder'), + + # smollm + TemplateInfo( + template_regex=f'.*{cases("smollm2")}.*', + modelfile_prefix= + 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/smollm2'), + TemplateInfo( + template_regex=f'.*{cases("smollm")}.*', + modelfile_prefix= + 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/smollm'), + + # 'aya' + TemplateInfo( + template_regex=f'.*{cases("aya-expanse")}.*', + modelfile_prefix= + 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/aya-expanse'), + TemplateInfo( + template_regex=f'.*{cases("aya")}.*', + modelfile_prefix= + 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/aya'), + + # falcon + TemplateInfo( + template_regex=f'.*{cases("falcon3")}.*', + modelfile_prefix= + 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/falcon3'), + TemplateInfo( + template_regex=f'.*{cases("falcon")}.*{cases("-2")}.*', + modelfile_prefix= + 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/falcon2'), + TemplateInfo( + template_regex=f'.*{cases("falcon")}.*', + modelfile_prefix= + 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/falcon'), + + # smallthinker + TemplateInfo( + template_regex=f'.*{cases("smallthinker")}.*', + modelfile_prefix= + 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/smallthinker'), + TemplateInfo( template_regex=f'.*{cases("nomic-embed-text")}.*', modelfile_prefix= @@ -651,10 +749,6 @@ template_info = [ template_regex=f'.*{cases("starcoder")}.*', modelfile_prefix= 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/starcoder'), - TemplateInfo( - template_regex=f'.*{cases("granite")}.*{cases("code")}.*', - modelfile_prefix= - 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/granite-code'), TemplateInfo( template_regex=f'.*{cases("all-minilm")}.*', modelfile_prefix= @@ -663,10 +757,6 @@ template_info = [ template_regex=f'.*{cases("openchat")}.*', modelfile_prefix= 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/openchat'), - TemplateInfo( - template_regex=f'.*{cases("aya")}.*', - modelfile_prefix= - 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/aya'), TemplateInfo( template_regex=f'.*{cases("openhermes")}.*', modelfile_prefix= @@ -687,10 +777,6 @@ template_info = [ template_regex=f'.*{cases("xwin")}.*{cases("lm")}.*', modelfile_prefix= 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/xwinlm'), - TemplateInfo( - template_regex=f'.*{cases("smollm")}.*', - modelfile_prefix= - 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/smollm'), TemplateInfo( template_regex=f'.*{cases("sqlcoder")}.*', modelfile_prefix= @@ -699,14 +785,6 @@ template_info = [ template_regex=f'.*{cases("starling-lm")}.*', modelfile_prefix= 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/starling-lm'), - TemplateInfo( - template_regex=f'.*{cases("falcon")}.*{cases("-2")}.*', - modelfile_prefix= - 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/falcon2'), - TemplateInfo( - template_regex=f'.*{cases("falcon")}.*', - modelfile_prefix= - 'https://modelscope.oss-cn-beijing.aliyuncs.com/llm_template/ollama/falcon'), TemplateInfo( template_regex=f'.*{cases("solar-pro")}.*', modelfile_prefix= diff --git a/tests/tools/test_to_ollama.py b/tests/tools/test_to_ollama.py index 78b885eb..3ef53727 100644 --- a/tests/tools/test_to_ollama.py +++ b/tests/tools/test_to_ollama.py @@ -311,6 +311,60 @@ class TestToOllama(unittest.TestCase): 'llama3.3') _test_check_tmpl_type('bartowski/EXAONE-3.5-7.8B-Instruct-GGUF', 'exaone3.5') + _test_check_tmpl_type( + 'QuantFactory/Tulu-3.1-8B-SuperNova-Smart-GGUF', + 'tulu3', + gguf_meta={'general.name': 'Tulu 3.1 8B SuperNova'}) + _test_check_tmpl_type( + 'bartowski/Athene-V2-Chat-GGUF', + 'athene-v2', + gguf_meta={'general.name': 'Athene V2 Chat'}) + _test_check_tmpl_type( + 'QuantFactory/granite-guardian-3.0-2b-GGUF', + 'granite3-guardian', + gguf_meta={'general.name': 'Models'}) + _test_check_tmpl_type('lmstudio-community/OpenCoder-8B-Instruct-GGUF', + 'opencoder') + _test_check_tmpl_type( + 'QuantFactory/SmolLM2-1.7B-Instruct-GGUF', + 'smollm2', + gguf_meta={'general.name': 'Smollm2 1.7B 8k Mix7 Ep2 v2'}) + _test_check_tmpl_type( + 'prithivMLmods/Aya-Expanse-8B-GGUF', + 'aya-expanse', + gguf_meta={'general.name': 'Aya Expanse 8b'}) + _test_check_tmpl_type('lmstudio-community/Falcon3-7B-Instruct-GGUF', + 'falcon3') + _test_check_tmpl_type( + 'lmstudio-community/granite-3.1-8b-instruct-GGUF', + 'granite3.1-dense', + gguf_meta={'general.name': 'Granite 3.1 8b Instruct'}) + _test_check_tmpl_type( + 'lmstudio-community/granite-3.1-2b-instruct-GGUF', + 'granite3.1-dense', + gguf_meta={'general.name': 'Granite 3.1 2b Instruct'}) + _test_check_tmpl_type( + 'lmstudio-community/granite-embedding-278m-multilingual-GGUF', + 'granite-embedding', + gguf_meta={'general.name': 'Granite Embedding 278m Multilingual'}) + _test_check_tmpl_type( + 'QuantFactory/granite-3.1-3b-a800m-instruct-GGUF', + 'granite3.1-moe', + gguf_meta={'general.name': 'Granite 3.1 3b A800M Base'}) + _test_check_tmpl_type( + 'bartowski/granite-3.1-1b-a400m-instruct-GGUF', + 'granite3.1-moe', + gguf_meta={'general.name': 'Granite 3.1 1b A400M Instruct'}) + _test_check_tmpl_type( + 'bartowski/SmallThinker-3B-Preview-GGUF', + 'smallthinker', + gguf_meta={'general.name': 'SmallThinker 3B Preview'}) + _test_check_tmpl_type( + 'bartowski/Dolphin3.0-Llama3.1-8B-GGUF', + 'dolphin3', + gguf_meta={'general.name': 'Dolphin 3.0 Llama 3.1 8B'}) + _test_check_tmpl_type( + 'AI-ModelScope/phi-4', 'phi4', gguf_meta={'general.name': 'Phi 4'}) if __name__ == '__main__': From cfd32abab2d5aec11a07c5b6bdd3e3500b23b0a8 Mon Sep 17 00:00:00 2001 From: Yunlin Mao Date: Fri, 10 Jan 2025 17:56:05 +0800 Subject: [PATCH 09/13] Unify datasets cache dir (#1178) * fix cache * fix lint * fix dataset cache * fix lint * remove --- modelscope/utils/config_ds.py | 9 +++++---- modelscope/utils/file_utils.py | 7 ++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/modelscope/utils/config_ds.py b/modelscope/utils/config_ds.py index 72a25887..eae85888 100644 --- a/modelscope/utils/config_ds.py +++ b/modelscope/utils/config_ds.py @@ -5,13 +5,14 @@ from pathlib import Path # Cache location from modelscope.hub.constants import DEFAULT_MODELSCOPE_DATA_ENDPOINT -from modelscope.utils.file_utils import get_modelscope_cache_dir +from modelscope.utils.file_utils import (get_dataset_cache_root, + get_modelscope_cache_dir) MS_CACHE_HOME = get_modelscope_cache_dir() -DEFAULT_MS_DATASETS_CACHE = os.path.join(MS_CACHE_HOME, 'hub', 'datasets') -MS_DATASETS_CACHE = Path( - os.getenv('MS_DATASETS_CACHE', DEFAULT_MS_DATASETS_CACHE)) +# NOTE: removed `MS_DATASETS_CACHE` env, +# default is `~/.cache/modelscope/hub/datasets` +MS_DATASETS_CACHE = get_dataset_cache_root() DOWNLOADED_DATASETS_DIR = 'downloads' DEFAULT_DOWNLOADED_DATASETS_PATH = os.path.join(MS_DATASETS_CACHE, diff --git a/modelscope/utils/file_utils.py b/modelscope/utils/file_utils.py index c00e8d26..50f66e34 100644 --- a/modelscope/utils/file_utils.py +++ b/modelscope/utils/file_utils.py @@ -60,11 +60,16 @@ def get_model_cache_root() -> str: def get_dataset_cache_root() -> str: """Get dataset raw file cache root path. + if `MODELSCOPE_CACHE` is set, return `MODELSCOPE_CACHE/datasets`, + else return `~/.cache/modelscope/hub/datasets` Returns: str: the modelscope dataset raw file cache root. """ - return os.path.join(get_modelscope_cache_dir(), 'datasets') + if os.getenv('MODELSCOPE_CACHE'): + return os.path.join(get_modelscope_cache_dir(), 'datasets') + else: + return os.path.join(get_modelscope_cache_dir(), 'hub', 'datasets') def get_dataset_cache_dir(dataset_id: str) -> str: From 72ccdb1a72e18ec1ad68314474fbe5f07fc2104c Mon Sep 17 00:00:00 2001 From: "Xingjun.Wang" Date: Fri, 10 Jan 2025 23:38:46 +0800 Subject: [PATCH 10/13] Add repo_id and repo_type in snapshot_download (#1172) * add repo_id and repo_type in snapshot_download * fix positional args * update --- modelscope/hub/snapshot_download.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/modelscope/hub/snapshot_download.py b/modelscope/hub/snapshot_download.py index 2f7f4790..31d1f091 100644 --- a/modelscope/hub/snapshot_download.py +++ b/modelscope/hub/snapshot_download.py @@ -19,6 +19,7 @@ from modelscope.hub.utils.utils import (get_model_masked_directory, model_id_to_group_owner_name) from modelscope.utils.constant import (DEFAULT_DATASET_REVISION, DEFAULT_MODEL_REVISION, + DEFAULT_REPOSITORY_REVISION, REPO_TYPE_DATASET, REPO_TYPE_MODEL, REPO_TYPE_SUPPORT) from modelscope.utils.logger import get_logger @@ -28,8 +29,8 @@ logger = get_logger() def snapshot_download( - model_id: str, - revision: Optional[str] = DEFAULT_MODEL_REVISION, + model_id: str = None, + revision: Optional[str] = None, cache_dir: Union[str, Path, None] = None, user_agent: Optional[Union[Dict, str]] = None, local_files_only: Optional[bool] = False, @@ -40,6 +41,8 @@ def snapshot_download( allow_patterns: Optional[Union[List[str], str]] = None, ignore_patterns: Optional[Union[List[str], str]] = None, max_workers: int = 8, + repo_id: str = None, + repo_type: Optional[str] = REPO_TYPE_MODEL, ) -> str: """Download all files of a repo. Downloads a whole snapshot of a repo's files at the specified revision. This @@ -51,7 +54,10 @@ def snapshot_download( user always has git and git-lfs installed, and properly configured. Args: - model_id (str): A user or an organization name and a repo name separated by a `/`. + repo_id (str): A user or an organization name and a repo name separated by a `/`. + model_id (str): A user or an organization name and a model name separated by a `/`. + if `repo_id` is provided, `model_id` will be ignored. + repo_type (str, optional): The type of the repo, either 'model' or 'dataset'. revision (str, optional): An optional Git revision id which can be a branch name, a tag, or a commit hash. NOTE: currently only branch and tag name is supported cache_dir (str, Path, optional): Path to the folder where cached files are stored, model will @@ -87,9 +93,22 @@ def snapshot_download( - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) if some parameter value is invalid """ + + repo_id = repo_id or model_id + if not repo_id: + raise ValueError('Please provide a valid model_id or repo_id') + + if repo_type not in REPO_TYPE_SUPPORT: + raise ValueError( + f'Invalid repo type: {repo_type}, only support: {REPO_TYPE_SUPPORT}' + ) + + if revision is None: + revision = DEFAULT_DATASET_REVISION if repo_type == REPO_TYPE_DATASET else DEFAULT_MODEL_REVISION + return _snapshot_download( - model_id, - repo_type=REPO_TYPE_MODEL, + repo_id, + repo_type=repo_type, revision=revision, cache_dir=cache_dir, user_agent=user_agent, From 262c2edd48c7eaff64e4b707ec106c1c2c749d53 Mon Sep 17 00:00:00 2001 From: suluyana <110878454+suluyana@users.noreply.github.com> Date: Mon, 13 Jan 2025 10:05:42 +0800 Subject: [PATCH 11/13] Fix/text gen (#1177) * fix text-gen: read pipeline type from configuration.json first --------- Co-authored-by: suluyan --- modelscope/pipelines/builder.py | 70 +++++++++++++++++++-------------- modelscope/utils/hub.py | 2 + 2 files changed, 42 insertions(+), 30 deletions(-) diff --git a/modelscope/pipelines/builder.py b/modelscope/pipelines/builder.py index 79459161..596d6d22 100644 --- a/modelscope/pipelines/builder.py +++ b/modelscope/pipelines/builder.py @@ -108,30 +108,7 @@ def pipeline(task: str = None, """ if task is None and pipeline_name is None: raise ValueError('task or pipeline_name is required') - prefer_llm_pipeline = kwargs.get('external_engine_for_llm') - if task is not None and task.lower() in [ - Tasks.text_generation, Tasks.chat - ]: - # if not specified, prefer llm pipeline for aforementioned tasks - if prefer_llm_pipeline is None: - prefer_llm_pipeline = True - # for llm pipeline, if llm_framework is not specified, default to swift instead - # TODO: port the swift infer based on transformer into ModelScope - if prefer_llm_pipeline and kwargs.get('llm_framework') is None: - kwargs['llm_framework'] = 'swift' - third_party = kwargs.get(ThirdParty.KEY) - if third_party is not None: - kwargs.pop(ThirdParty.KEY) - if pipeline_name is None and prefer_llm_pipeline: - pipeline_name = external_engine_for_llm_checker( - model, model_revision, kwargs) - if pipeline_name is None: - model = normalize_model_input( - model, - model_revision, - third_party=third_party, - ignore_file_pattern=ignore_file_pattern) - pipeline_props = {'type': pipeline_name} + if pipeline_name is None: # get default pipeline for this task if isinstance(model, str) \ @@ -142,16 +119,47 @@ def pipeline(task: str = None, model, revision=model_revision) if isinstance( model, str) else read_config( model[0], revision=model_revision) - register_plugins_repo(cfg.safe_get('plugins')) - register_modelhub_repo(model, cfg.get('allow_remote', False)) - pipeline_name = external_engine_for_llm_checker( - model, model_revision, - kwargs) if prefer_llm_pipeline else None - if pipeline_name is not None: + if cfg: + pipeline_name = cfg.safe_get('pipeline', + {}).get('type', None) + + if pipeline_name is None: + prefer_llm_pipeline = kwargs.get('external_engine_for_llm') + # if not specified in both args and configuration.json, prefer llm pipeline for aforementioned tasks + if task is not None and task.lower() in [ + Tasks.text_generation, Tasks.chat + ]: + if prefer_llm_pipeline is None: + prefer_llm_pipeline = True + # for llm pipeline, if llm_framework is not specified, default to swift instead + # TODO: port the swift infer based on transformer into ModelScope + if prefer_llm_pipeline: + if kwargs.get('llm_framework') is None: + kwargs['llm_framework'] = 'swift' + pipeline_name = external_engine_for_llm_checker( + model, model_revision, kwargs) + + if pipeline_name is None or pipeline_name != 'llm': + third_party = kwargs.get(ThirdParty.KEY) + if third_party is not None: + kwargs.pop(ThirdParty.KEY) + + model = normalize_model_input( + model, + model_revision, + third_party=third_party, + ignore_file_pattern=ignore_file_pattern) + + register_plugins_repo(cfg.safe_get('plugins')) + register_modelhub_repo(model, + cfg.get('allow_remote', False)) + + if pipeline_name: pipeline_props = {'type': pipeline_name} else: check_config(cfg) pipeline_props = cfg.pipeline + elif model is not None: # get pipeline info from Model object first_model = model[0] if isinstance(model, list) else model @@ -165,6 +173,8 @@ def pipeline(task: str = None, pipeline_name, default_model_repo = get_default_pipeline_info(task) model = normalize_model_input(default_model_repo, model_revision) pipeline_props = {'type': pipeline_name} + else: + pipeline_props = {'type': pipeline_name} pipeline_props['model'] = model pipeline_props['device'] = device diff --git a/modelscope/utils/hub.py b/modelscope/utils/hub.py index 39ca644a..79df8545 100644 --- a/modelscope/utils/hub.py +++ b/modelscope/utils/hub.py @@ -54,6 +54,8 @@ def read_config(model_id_or_path: str, local_path = os.path.join(model_id_or_path, ModelFile.CONFIGURATION) elif os.path.isfile(model_id_or_path): local_path = model_id_or_path + else: + return None return Config.from_file(local_path) From e4a51f5960c7a1a061421064ef9163f76d4d6a41 Mon Sep 17 00:00:00 2001 From: Yunlin Mao Date: Mon, 13 Jan 2025 21:03:59 +0800 Subject: [PATCH 12/13] update doc with llama_index (#1180) --- ...rch_QA_based_on_langchain_llamaindex.ipynb | 342 +++++++++++++++++- ...en_doc_search_QA_based_on_llamaindex.ipynb | 126 +++++-- 2 files changed, 422 insertions(+), 46 deletions(-) diff --git a/examples/pytorch/application/qwen_doc_search_QA_based_on_langchain_llamaindex.ipynb b/examples/pytorch/application/qwen_doc_search_QA_based_on_langchain_llamaindex.ipynb index e6ddabfd..a66c079d 100644 --- a/examples/pytorch/application/qwen_doc_search_QA_based_on_langchain_llamaindex.ipynb +++ b/examples/pytorch/application/qwen_doc_search_QA_based_on_langchain_llamaindex.ipynb @@ -2,6 +2,10 @@ "cells": [ { "cell_type": "markdown", + "id": "8230365523c9330a", + "metadata": { + "collapsed": false + }, "source": [ "# Usage\n", "1. Install python dependencies\n", @@ -30,11 +34,7 @@ "``` \n", "\n", "3. Enjoy your QA AI" - ], - "metadata": { - "collapsed": false - }, - "id": "8230365523c9330a" + ] }, { "cell_type": "code", @@ -94,7 +94,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "1cb8feca-c71f-4ad6-8eff-caae95411aa0", "metadata": { "ExecutionIndicator": { @@ -109,7 +109,312 @@ }, "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloading Model to directory: /mnt/workspace/.cache/modelscope/Qwen/Qwen-1_8B-Chat\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloading Model to directory: /mnt/workspace/.cache/modelscope/Qwen/Qwen-1_8B-Chat\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Try importing flash-attention for faster inference...\n", + "Warning: import flash_attn rotary fail, please install FlashAttention rotary to get higher efficiency https://github.com/Dao-AILab/flash-attention/tree/main/csrc/rotary\n", + "Warning: import flash_attn rms_norm fail, please install FlashAttention layer_norm to get higher efficiency https://github.com/Dao-AILab/flash-attention/tree/main/csrc/layer_norm\n", + "Warning: import flash_attn fail, please install FlashAttention to get higher efficiency https://github.com/Dao-AILab/flash-attention\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "114cb3d66e9e4f6694ba66c91fc4b8a9", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Loading checkpoint shards: 0%| | 0/2 [00:00 List[Document]:\n", " \"\"\"Get documents relevant for a query.\"\"\"\n", " try:\n", - " from llama_index.indices.base import BaseIndex\n", - " from llama_index.response.schema import Response\n", + " from llama_index.core.indices.base import BaseIndex\n", + " from llama_index.core import Response\n", " except ImportError:\n", " raise ImportError(\n", " \"You need to install `pip install llama-index` to use this retriever.\"\n", @@ -239,9 +541,9 @@ "\n", "# define QianWen LLM based on langchain's LLM to use models in Modelscope\n", "class QianWenChatLLM(LLM):\n", - " max_length = 10000\n", + " max_length: int = 10000\n", " temperature: float = 0.01\n", - " top_p = 0.9\n", + " top_p: float = 0.9\n", "\n", " def __init__(self):\n", " super().__init__()\n", @@ -270,11 +572,11 @@ "# STEP2: load knowledge file and initialize vector db by llamaIndex\n", "print('STEP2: reading docs ...')\n", "embeddings = ModelScopeEmbeddings4LlamaIndex(model_id=embedding_model)\n", - "service_context = ServiceContext.from_defaults(embed_model=embeddings, llm=None)\n", - "set_global_service_context(service_context) # global config, not good\n", + "Settings.llm = None\n", + "Settings.embed_model=embeddings # global config, not good\n", "\n", "llamaIndex_docs = SimpleDirectoryReader(knowledge_doc_file_dir).load_data()\n", - "llamaIndex_index = GPTVectorStoreIndex.from_documents(llamaIndex_docs, chunk_size=512)\n", + "llamaIndex_index = VectorStoreIndex.from_documents(llamaIndex_docs, chunk_size=512)\n", "retriever = LlamaIndexRetriever(index=llamaIndex_index)\n", "print(' 2.2 reading doc done, vec db created.')\n", "\n", @@ -318,7 +620,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.9.20" } }, "nbformat": 4, diff --git a/examples/pytorch/application/qwen_doc_search_QA_based_on_llamaindex.ipynb b/examples/pytorch/application/qwen_doc_search_QA_based_on_llamaindex.ipynb index 194c46a2..51505e84 100644 --- a/examples/pytorch/application/qwen_doc_search_QA_based_on_llamaindex.ipynb +++ b/examples/pytorch/application/qwen_doc_search_QA_based_on_llamaindex.ipynb @@ -2,6 +2,10 @@ "cells": [ { "cell_type": "markdown", + "id": "f4abc589d9bfffca", + "metadata": { + "collapsed": false + }, "source": [ "# Usage\n", "\n", @@ -34,27 +38,29 @@ "```\n", "\n", "## 3. Go!" - ], - "metadata": { - "collapsed": false - }, - "id": "f4abc589d9bfffca" + ] }, { "cell_type": "code", + "execution_count": null, + "id": "c32122833dd7b8c8", + "metadata": { + "collapsed": false + }, "outputs": [], "source": [ "!pip install modelscope\n", "!pip install transformers -U\n", "!pip install llama-index llama-index-llms-huggingface ipywidgets " - ], - "metadata": { - "collapsed": false - }, - "id": "c32122833dd7b8c8" + ] }, { "cell_type": "code", + "execution_count": null, + "id": "63704e2b21a9ba52", + "metadata": { + "collapsed": false + }, "outputs": [], "source": [ "!wget https://modelscope.oss-cn-beijing.aliyuncs.com/resource/rag/punkt.zip\n", @@ -74,15 +80,90 @@ "!mv /mnt/workspace/xianjiaoda.md /mnt/workspace/custom_data\n", "\n", "!cd /mnt/workspace" - ], - "metadata": { - "collapsed": false - }, - "id": "63704e2b21a9ba52" + ] }, { "cell_type": "code", - "outputs": [], + "execution_count": 2, + "id": "eef67659e94045c5", + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloading Model to directory: /mnt/workspace/.cache/modelscope/qwen/Qwen1.5-4B-Chat\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-01-13 15:52:53,260 - modelscope - INFO - Model revision not specified, using default: [master] version.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-01-13 15:52:53,637 - modelscope - INFO - Creating symbolic link [/mnt/workspace/.cache/modelscope/qwen/Qwen1.5-4B-Chat].\n", + "2025-01-13 15:52:53,638 - modelscope - WARNING - Failed to create symbolic link /mnt/workspace/.cache/modelscope/qwen/Qwen1.5-4B-Chat for /mnt/workspace/.cache/modelscope/qwen/Qwen1___5-4B-Chat.\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4523c5dd31ba411d95cc0ce9e5da8ded", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Loading checkpoint shards: 0%| | 0/2 [00:00 Date: Wed, 15 Jan 2025 00:07:31 +0800 Subject: [PATCH 13/13] update version to 1.22.1 --- modelscope/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modelscope/version.py b/modelscope/version.py index 2ace39be..4910ad4b 100644 --- a/modelscope/version.py +++ b/modelscope/version.py @@ -1,5 +1,5 @@ # Make sure to modify __release_datetime__ to release time when making official release. -__version__ = '1.22.0' +__version__ = '1.22.1' # default release datetime for branches under active development is set # to be a time far-far-away-into-the-future -__release_datetime__ = '2025-01-02 23:59:59' +__release_datetime__ = '2025-01-15 01:00:00'