diff --git a/modelscope/metainfo.py b/modelscope/metainfo.py index b5ec251a..23ffdab1 100644 --- a/modelscope/metainfo.py +++ b/modelscope/metainfo.py @@ -563,6 +563,7 @@ class Pipelines(object): efficient_diffusion_tuning = 'efficient-diffusion-tuning' multimodal_dialogue = 'multimodal-dialogue' llama2_text_generation_pipeline = 'llama2-text-generation-pipeline' + llama2_text_generation_chat_pipeline = 'llama2-text-generation-chat-pipeline' image_to_video_task_pipeline = 'image-to-video-task-pipeline' video_to_video_pipeline = 'video-to-video-pipeline' diff --git a/modelscope/models/nlp/llama/text_generation.py b/modelscope/models/nlp/llama/text_generation.py index dab0f757..b9cc8032 100644 --- a/modelscope/models/nlp/llama/text_generation.py +++ b/modelscope/models/nlp/llama/text_generation.py @@ -72,6 +72,7 @@ def get_chat_prompt(system: str, text: str, history: List[Tuple[str, str]], # This file is mainly copied from the llama code of transformers @MODELS.register_module(Tasks.text_generation, module_name=Models.llama2) +@MODELS.register_module(Tasks.chat, module_name=Models.llama2) @MODELS.register_module(Tasks.text_generation, module_name=Models.llama) class LlamaForTextGeneration(MsModelMixin, LlamaForCausalLM, TorchModel): diff --git a/modelscope/pipelines/builder.py b/modelscope/pipelines/builder.py index ca1431ea..d6dff693 100644 --- a/modelscope/pipelines/builder.py +++ b/modelscope/pipelines/builder.py @@ -19,7 +19,10 @@ from .util import is_official_hub_path PIPELINES = Registry('pipelines') -def normalize_model_input(model, model_revision, third_party=None): +def normalize_model_input(model, + model_revision, + third_party=None, + ignore_file_pattern=None): """ normalize the input model, to ensure that a model str is a valid local path: in other words, for model represented by a model id, the model shall be downloaded locally """ @@ -31,7 +34,10 @@ def normalize_model_input(model, model_revision, third_party=None): if third_party is not None: user_agent[ThirdParty.KEY] = third_party model = snapshot_download( - model, revision=model_revision, user_agent=user_agent) + model, + revision=model_revision, + user_agent=user_agent, + ignore_file_pattern=ignore_file_pattern) elif isinstance(model, list) and isinstance(model[0], str): for idx in range(len(model)): if is_official_hub_path( @@ -68,6 +74,7 @@ def pipeline(task: str = None, framework: str = None, device: str = 'gpu', model_revision: Optional[str] = DEFAULT_MODEL_REVISION, + ignore_file_pattern: List[str] = None, **kwargs) -> Pipeline: """ Factory method to build an obj:`Pipeline`. @@ -82,6 +89,8 @@ def pipeline(task: str = None, model_revision: revision of model(s) if getting from model hub, for multiple models, expecting all models to have the same revision device (str, optional): whether to use gpu or cpu is used to do inference. + ignore_file_pattern(`str` or `List`, *optional*, default to `None`): + Any file pattern to be ignored in downloading, like exact file names or file extensions. Return: pipeline (obj:`Pipeline`): pipeline object for certain task. @@ -104,7 +113,10 @@ def pipeline(task: str = None, if third_party is not None: kwargs.pop(ThirdParty.KEY) model = normalize_model_input( - model, model_revision, third_party=third_party) + model, + model_revision, + third_party=third_party, + ignore_file_pattern=ignore_file_pattern) pipeline_props = {'type': pipeline_name} if pipeline_name is None: # get default pipeline for this task diff --git a/modelscope/pipelines/nlp/text_generation_pipeline.py b/modelscope/pipelines/nlp/text_generation_pipeline.py index 37396105..779c8a54 100644 --- a/modelscope/pipelines/nlp/text_generation_pipeline.py +++ b/modelscope/pipelines/nlp/text_generation_pipeline.py @@ -2,7 +2,7 @@ # Copyright (c) 2022 Zhipu.AI import os -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, List, Optional, Union import torch from transformers import GenerationConfig @@ -450,7 +450,7 @@ class SeqGPTPipeline(Pipeline): # gen & decode # prompt += '[GEN]' input_ids = self.tokenizer( - prompt + '[GEN]', + prompt + forward_params.get('gen_token', ''), return_tensors='pt', padding=True, truncation=True, @@ -519,15 +519,15 @@ class Llama2TaskPipeline(TextGenerationPipeline): return {}, pipeline_parameters, {} def forward(self, - inputs, - max_length=2048, - do_sample=True, - top_p=0.85, - temperature=1.0, - repetition_penalty=1., - eos_token_id=2, - bos_token_id=1, - pad_token_id=0, + inputs: str, + max_length: int = 2048, + do_sample: bool = False, + top_p: float = 0.9, + temperature: float = 0.6, + repetition_penalty: float = 1., + eos_token_id: int = 2, + bos_token_id: int = 1, + pad_token_id: int = 0, **forward_params) -> Dict[str, Any]: output = {} inputs = self.tokenizer( @@ -553,3 +553,96 @@ class Llama2TaskPipeline(TextGenerationPipeline): # format the outputs from pipeline def postprocess(self, input, **kwargs) -> Dict[str, Any]: return input + + +@PIPELINES.register_module( + Tasks.chat, module_name=Pipelines.llama2_text_generation_chat_pipeline) +class Llama2chatTaskPipeline(Pipeline): + """Use `model` and `preprocessor` to create a generation pipeline for prediction. + + Args: + model (str or Model): Supply either a local model dir which supported the text generation task, + or a model id from the model hub, or a torch model instance. + preprocessor (Preprocessor): An optional preprocessor instance, please make sure the preprocessor fits for + the model if supplied. + kwargs (dict, `optional`): + Extra kwargs passed into the preprocessor's constructor. + Examples: + >>> from modelscope.utils.constant import Tasks + >>> import torch + >>> from modelscope.pipelines import pipeline + >>> from modelscope import Model + >>> pipe = pipeline(task=Tasks.chat, model="modelscope/Llama-2-7b-chat-ms", device_map='auto', + >>> torch_dtype=torch.float16, ignore_file_pattern = [r'.+\\.bin$'], model_revision='v1.0.5') + >>> inputs = 'Where is the capital of Zhejiang?' + >>> result = pipe(inputs,max_length=512, do_sample=False, top_p=0.9, + >>> temperature=0.6, repetition_penalty=1., eos_token_id=2, bos_token_id=1, pad_token_id=0) + >>> print(result['response']) + >>> inputs = 'What are the interesting places there?' + >>> result = pipe(inputs,max_length=512, do_sample=False, top_p=0.9, + >>> temperature=0.6, repetition_penalty=1., eos_token_id=2, bos_token_id=1, + >>> pad_token_id=0, history=result['history']) + >>> print(result['response']) + >>> inputs = 'What are the company there?' + >>> history_demo = [('Where is the capital of Zhejiang?', + >>> 'Thank you for asking! The capital of Zhejiang Province is Hangzhou.')] + >>> result = pipe(inputs,max_length=512, do_sample=False, top_p=0.9, + >>> temperature=0.6, repetition_penalty=1., eos_token_id=2, bos_token_id=1, + >>> pad_token_id=0, history=history_demo) + >>> print(result['response']) + + To view other examples plese check tests/pipelines/test_llama2_text_generation_pipeline.py. + """ + + def __init__(self, + model: Union[Model, str], + preprocessor: Preprocessor = None, + config_file: str = None, + device: str = 'gpu', + auto_collate: bool = True, + **kwargs) -> Dict[str, Any]: + device_map = kwargs.get('device_map', None) + torch_dtype = kwargs.get('torch_dtype', None) + self.model = Model.from_pretrained( + model, device_map=device_map, torch_dtype=torch_dtype) + from modelscope.models.nlp.llama2 import Llama2Tokenizer + self.tokenizer = Llama2Tokenizer.from_pretrained(model) + super().__init__(model=self.model, **kwargs) + + def preprocess(self, inputs, **preprocess_params) -> Dict[str, Any]: + return inputs + + def _sanitize_parameters(self, **pipeline_parameters): + return {}, pipeline_parameters, {} + + def forward(self, + inputs: str, + max_length: int = 2048, + do_sample: bool = False, + top_p: float = 0.9, + temperature: float = 0.6, + repetition_penalty: float = 1., + eos_token_id: int = 2, + bos_token_id: int = 1, + pad_token_id: int = 0, + system: str = 'you are a helpful assistant!', + history: List = [], + **forward_params) -> Dict[str, Any]: + inputs_dict = forward_params + inputs_dict['text'] = inputs + inputs_dict['max_length'] = max_length + inputs_dict['do_sample'] = do_sample + inputs_dict['top_p'] = top_p + inputs_dict['temperature'] = temperature + inputs_dict['repetition_penalty'] = repetition_penalty + inputs_dict['eos_token_id'] = eos_token_id + inputs_dict['bos_token_id'] = bos_token_id + inputs_dict['pad_token_id'] = pad_token_id + inputs_dict['system'] = system + inputs_dict['history'] = history + output = self.model.chat(inputs_dict, self.tokenizer) + return output + + # format the outputs from pipeline + def postprocess(self, input, **kwargs) -> Dict[str, Any]: + return input diff --git a/tests/pipelines/test_llama2_text_generation_pipeline.py b/tests/pipelines/test_llama2_text_generation_pipeline.py index 2a532257..a9db6acf 100644 --- a/tests/pipelines/test_llama2_text_generation_pipeline.py +++ b/tests/pipelines/test_llama2_text_generation_pipeline.py @@ -13,18 +13,21 @@ class Llama2TextGenerationPipelineTest(unittest.TestCase): def setUp(self) -> None: self.llama2_model_id_7B_chat_ms = 'modelscope/Llama-2-7b-chat-ms' - self.llama2_input_chat_ch = '天空为什么是蓝色的？' + self.llama2_input_chat_ch = 'What are the company there?' + self.history_demo = [( + 'Where is the capital of Zhejiang?', + 'Thank you for asking! The capital of Zhejiang Province is Hangzhou.' + )] def run_pipeline_with_model_id(self, model_id, input, init_kwargs={}, run_kwargs={}): - pipeline_ins = pipeline( - task=Tasks.text_generation, model=model_id, **init_kwargs) + pipeline_ins = pipeline(task=Tasks.chat, model=model_id, **init_kwargs) pipeline_ins._model_prepare = True result = pipeline_ins(input, **run_kwargs) - print(result['text']) + print(result['response']) # 7B_ms_chat @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') @@ -34,12 +37,15 @@ class Llama2TextGenerationPipelineTest(unittest.TestCase): self.llama2_input_chat_ch, init_kwargs={ 'device_map': 'auto', - 'torch_dtype': torch.float16 + 'torch_dtype': torch.float16, + 'model_revision': 'v1.0.5', + 'ignore_file_pattern': [r'.+\.bin$'] }, run_kwargs={ - 'max_length': 200, + 'max_length': 512, 'do_sample': True, - 'top_p': 0.85 + 'top_p': 0.9, + 'history': self.history_demo }) diff --git a/tests/pipelines/test_text_generation.py b/tests/pipelines/test_text_generation.py index 2197c4d7..b82be76b 100644 --- a/tests/pipelines/test_text_generation.py +++ b/tests/pipelines/test_text_generation.py @@ -46,7 +46,7 @@ class TextGenerationTest(unittest.TestCase): self.llama_model_id = 'skyline2006/llama-7b' self.llama_input = 'My name is Merve and my favorite' self.seqgpt_model_id = 'damo/nlp_seqgpt-560m' - self.ecomgpt_model_id = 'damo/nlp_seqgpt-560m' + self.ecomgpt_model_id = 'damo/nlp_ecomgpt_multilingual-7B-ecom' def run_pipeline_with_model_instance(self, model_id, input): model = Model.from_pretrained(model_id) @@ -327,7 +327,8 @@ class TextGenerationTest(unittest.TestCase): inputs = {'task': '抽取', 'text': '杭州欢迎你。', 'labels': '地名'} PROMPT_TEMPLATE = '输入: {text}\n{task}: {labels}\n输出: ' prompt = PROMPT_TEMPLATE.format(**inputs) - self.run_pipeline_with_model_id(self.seqgpt_model_id, prompt) + self.run_pipeline_with_model_id( + self.seqgpt_model_id, prompt, run_kwargs={'gen_token': '[GEN]'}) @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_ecomgpt_with_model_name(self): @@ -336,7 +337,7 @@ class TextGenerationTest(unittest.TestCase): '### Instruction:\n{text}\n{instruction}\n\n### Response:' inputs = { 'instruction': - 'Classify the sentence, candidate labels: product, brand', + 'Classify the sentence, select from the candidate labels: product, brand', 'text': '照相机' } prompt = PROMPT_TEMPLATE.format(**inputs)