From bacdc53327213e13e9585d08598befcdb4637e55 Mon Sep 17 00:00:00 2001 From: suluyana <110878454+suluyana@users.noreply.github.com> Date: Thu, 17 Oct 2024 17:13:22 +0800 Subject: [PATCH 1/4] fix: set device_map=cuda if using gptq in gpu (#1027) --- .../msdatasets/context/dataset_context_config.py | 3 ++- modelscope/pipelines/nlp/llm_pipeline.py | 16 ++++++++++++---- tests/msdatasets/test_general_datasets.py | 3 ++- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/modelscope/msdatasets/context/dataset_context_config.py b/modelscope/msdatasets/context/dataset_context_config.py index bfe7dbd1..fce262b0 100644 --- a/modelscope/msdatasets/context/dataset_context_config.py +++ b/modelscope/msdatasets/context/dataset_context_config.py @@ -17,7 +17,8 @@ class DatasetContextConfig: data_files: Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]], download_mode: DownloadMode, cache_root_dir: str, - use_streaming: bool, stream_batch_size: int, trust_remote_code: bool, **kwargs): + use_streaming: bool, stream_batch_size: int, + trust_remote_code: bool, **kwargs): self._download_config = None self._data_meta_config = None diff --git a/modelscope/pipelines/nlp/llm_pipeline.py b/modelscope/pipelines/nlp/llm_pipeline.py index 9e0c6241..cb801bd4 100644 --- a/modelscope/pipelines/nlp/llm_pipeline.py +++ b/modelscope/pipelines/nlp/llm_pipeline.py @@ -7,7 +7,7 @@ from typing import Any, Callable, Dict, Generator, Iterator, List, Tuple, Union import json import numpy as np import torch -from transformers import PreTrainedModel, PreTrainedTokenizer +from transformers import AutoConfig, PreTrainedModel, PreTrainedTokenizer from modelscope import (AutoModelForCausalLM, AutoTokenizer, Pipeline, snapshot_download) @@ -170,9 +170,17 @@ class LLMPipeline(Pipeline, PipelineStreamingOutputMixin): **kwargs): self.device_map = kwargs.pop('device_map', None) self.llm_framework = llm_framework - # TODO: qwen-int4 need 'cuda'/'auto' device_map. - if not self.device_map and 'qwen' in kwargs['model'].lower(): - self.device_map = 'cuda' + + if os.path.exists(kwargs['model']): + config = AutoConfig.from_pretrained( + kwargs['model'], trust_remote_code=True) + q_config = config.__dict__.get('quantization_config', None) + if q_config: + if q_config.get( + 'quant_method', + 'gptq') == 'gptq' and torch.cuda.device_count(): + self.device_map = 'cuda' + self.torch_dtype = kwargs.pop('torch_dtype', None) self.ignore_file_pattern = kwargs.pop('ignore_file_pattern', None) diff --git a/tests/msdatasets/test_general_datasets.py b/tests/msdatasets/test_general_datasets.py index 0792cf46..1fd96f8f 100644 --- a/tests/msdatasets/test_general_datasets.py +++ b/tests/msdatasets/test_general_datasets.py @@ -121,7 +121,8 @@ class GeneralMsDatasetTest(unittest.TestCase): f'please check the url: {py_script_url}' # Load the dataset - ds = MsDataset.load(py_script_file, subset_name='cola', split='train') + ds = MsDataset.load( + py_script_file, subset_name='cola', split='train') sample = next(iter(ds)) logger.info(f'>>output of test_local_py_script:\n {sample}') assert sample From 88aefcdd746536362ab7a43d617a3af9c754c007 Mon Sep 17 00:00:00 2001 From: tastelikefeet <58414341+tastelikefeet@users.noreply.github.com> Date: Thu, 17 Oct 2024 18:35:08 +0800 Subject: [PATCH 2/4] lint code (#1025) From 607b7c37fe36aa8be7ecb62c8444ec31a688538d Mon Sep 17 00:00:00 2001 From: tastelikefeet <58414341+tastelikefeet@users.noreply.github.com> Date: Mon, 21 Oct 2024 10:28:51 +0800 Subject: [PATCH 3/4] Fix patch processors and feature extractors (#1031) --- modelscope/utils/hf_util.py | 48 +++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/modelscope/utils/hf_util.py b/modelscope/utils/hf_util.py index f5fb8d33..31a061cc 100644 --- a/modelscope/utils/hf_util.py +++ b/modelscope/utils/hf_util.py @@ -6,6 +6,7 @@ from types import MethodType from typing import Dict, Literal, Optional, Union from transformers import AutoConfig as AutoConfigHF +from transformers import AutoFeatureExtractor as AutoFeatureExtractorHF from transformers import AutoImageProcessor as AutoImageProcessorHF from transformers import AutoModel as AutoModelHF from transformers import AutoModelForCausalLM as AutoModelForCausalLMHF @@ -14,6 +15,7 @@ from transformers import \ AutoModelForSequenceClassification as AutoModelForSequenceClassificationHF from transformers import \ AutoModelForTokenClassification as AutoModelForTokenClassificationHF +from transformers import AutoProcessor as AutoProcessorHF from transformers import AutoTokenizer as AutoTokenizerHF from transformers import BatchFeature as BatchFeatureHF from transformers import BitsAndBytesConfig as BitsAndBytesConfigHF @@ -161,6 +163,7 @@ def _patch_pretrained_class(): ignore_file_pattern, **kwargs) return ori_get_config_dict(cls, model_dir, **kwargs) + PretrainedConfig.from_pretrained = from_pretrained PretrainedConfig.get_config_dict = get_config_dict def patch_model_base(): @@ -177,9 +180,54 @@ def _patch_pretrained_class(): PreTrainedModel.from_pretrained = from_pretrained + def patch_image_processor_base(): + """ Monkey patch AutoImageProcessorHF.from_pretrained to adapt to modelscope hub. + """ + ori_from_pretrained = AutoImageProcessorHF.from_pretrained.__func__ + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, + **kwargs): + model_dir = get_model_dir(pretrained_model_name_or_path, None, + **kwargs) + return ori_from_pretrained(cls, model_dir, *model_args, **kwargs) + + AutoImageProcessorHF.from_pretrained = from_pretrained + + def patch_auto_processor_base(): + """ Monkey patch AutoProcessorHF.from_pretrained to adapt to modelscope hub. + """ + ori_from_pretrained = AutoProcessorHF.from_pretrained.__func__ + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, + **kwargs): + model_dir = get_model_dir(pretrained_model_name_or_path, None, + **kwargs) + return ori_from_pretrained(cls, model_dir, *model_args, **kwargs) + + AutoProcessorHF.from_pretrained = from_pretrained + + def patch_feature_extractor_base(): + """ Monkey patch AutoFeatureExtractorHF.from_pretrained to adapt to modelscope hub. + """ + ori_from_pretrained = AutoFeatureExtractorHF.from_pretrained.__func__ + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, + **kwargs): + model_dir = get_model_dir(pretrained_model_name_or_path, None, + **kwargs) + return ori_from_pretrained(cls, model_dir, *model_args, **kwargs) + + AutoFeatureExtractorHF.from_pretrained = from_pretrained + patch_tokenizer_base() patch_config_base() patch_model_base() + patch_image_processor_base() + patch_auto_processor_base() + patch_feature_extractor_base() def patch_hub(): From e320b40a0dd77e6234eb65af7ac241b5eb8c723f Mon Sep 17 00:00:00 2001 From: "Xingjun.Wang" Date: Mon, 21 Oct 2024 11:25:02 +0800 Subject: [PATCH 4/4] Fix/dockerfile (#1034) * update version * update build_image for swift * add outlines in build_image.sh * set outlines to 0.0.46 to avoid failure of building docker * set vllm==0.5.2 in Dockerfile.ubuntu to fix CI issue * update outlines version: <0.1 * update ms-swift installation in build_image.sh * merge diff from release/1.19 --- .dev_scripts/build_image.sh | 2 +- docker/Dockerfile.ubuntu | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.dev_scripts/build_image.sh b/.dev_scripts/build_image.sh index d281b786..f22d70cd 100644 --- a/.dev_scripts/build_image.sh +++ b/.dev_scripts/build_image.sh @@ -159,7 +159,7 @@ docker_file_content=`cat docker/Dockerfile.ubuntu` BUILD_HASH_ID=$(git rev-parse HEAD) # install thrid part library -docker_file_content="${docker_file_content} \nRUN export COMMIT_ID=$BUILD_HASH_ID && pip install --no-cache-dir -U adaseq pai-easycv && pip install --no-cache-dir -U 'ms-swift' 'decord' 'qwen_vl_utils' 'pyav' 'librosa' 'funasr' autoawq 'timm>0.9.5' 'transformers' 'accelerate' 'peft' 'optimum' 'trl'" +docker_file_content="${docker_file_content} \nRUN export COMMIT_ID=$BUILD_HASH_ID && pip install --no-cache-dir -U adaseq pai-easycv && pip install --no-cache-dir -U 'ms-swift' 'decord' 'qwen_vl_utils' 'pyav' 'librosa' 'funasr' autoawq 'timm>0.9.5' 'transformers' 'accelerate' 'peft' 'optimum' 'trl' 'outlines<0.1'" docker_file_content="${docker_file_content} \nRUN pip uninstall modelscope -y && export COMMIT_ID=$BUILD_HASH_ID && cd /tmp && GIT_LFS_SKIP_SMUDGE=1 git clone -b $build_branch --single-branch $REPO_URL && cd modelscope && pip install . && cd / && rm -fr /tmp/modelscope && pip cache purge;" diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu index ed7c0cba..ca6d5e50 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.ubuntu @@ -58,7 +58,7 @@ RUN if [ "$USE_GPU" = "True" ] ; then \ pip install --no-cache-dir -U 'xformers<0.0.27' --index-url https://download.pytorch.org/whl/cu121 && \ pip install --no-cache-dir --force tinycudann==1.7 -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html && \ pip uninstall -y torch-scatter && TORCH_CUDA_ARCH_LIST="6.0;6.1;6.2;7.0;7.5;8.0;8.6;8.9;9.0" pip install --no-cache-dir -U torch-scatter && \ - pip install --no-cache-dir -U triton 'vllm==0.5.1' https://modelscope.oss-cn-beijing.aliyuncs.com/packages/lmdeploy-0.5.0-cp310-cp310-linux_x86_64.whl; \ + pip install --no-cache-dir -U triton 'vllm==0.5.2' https://modelscope.oss-cn-beijing.aliyuncs.com/packages/lmdeploy-0.5.0-cp310-cp310-linux_x86_64.whl; \ else \ echo 'cpu unsupport vllm auto-gptq'; \ fi