[to #42322933] add polylm, a polyglot large language model

Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/13339595

modelscope/metainfo.py

@@ -148,6 +148,7 @@ class Models(object):
     bert_for_ds = 'bert-for-document-segmentation'
     ponet_for_ds = 'ponet-for-document-segmentation'
     ponet = 'ponet'
+    polylm = 'polylm'
     T5 = 'T5'
     mglm = 'mglm'
     codegeex = 'codegeex'
@@ -445,6 +446,7 @@ class Pipelines(object):
     word_alignment = 'word-alignment'
     plug_generation = 'plug-generation'
     gpt3_generation = 'gpt3-generation'
+    polylm_text_generation = 'polylm-text-generation'
     gpt_moe_generation = 'gpt-moe-generation'
     faq_question_answering = 'faq-question-answering'
     conversational_text_to_sql = 'conversational-text-to-sql'

modelscope/models/nlp/__init__.py

@@ -21,6 +21,7 @@ if TYPE_CHECKING:
     from .glm_130b import GLM130bForTextGeneration
     from .csanmt import CsanmtForTranslation
     from .canmt import CanmtForTranslation
+    from .polylm import PolyLMForTextGeneration
     from .deberta_v2 import DebertaV2ForMaskedLM, DebertaV2Model
     from .chatglm import ChatGLMForConditionalGeneration, ChatGLMTokenizer, ChatGLMConfig
     from .chatglm2 import ChatGLM2ForConditionalGeneration, ChatGLM2Tokenizer, ChatGLM2Config
@@ -93,6 +94,7 @@ else:
         'bloom': ['BloomModel'],
         'csanmt': ['CsanmtForTranslation'],
         'canmt': ['CanmtForTranslation'],
+        'polylm': ['PolyLMForTextGeneration'],
         'codegeex':
         ['CodeGeeXForCodeTranslation', 'CodeGeeXForCodeGeneration'],
         'glm_130b': ['GLM130bForTextGeneration'],

modelscope/models/nlp/polylm/__init__.py

@@ -0,0 +1,19 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import TYPE_CHECKING
+
+from modelscope.utils.import_utils import LazyImportModule
+
+if TYPE_CHECKING:
+    from .text_generation import PolyLMForTextGeneration
+else:
+    _import_structure = {'text_generation': ['PolyLMForTextGeneration']}
+
+    import sys
+
+    sys.modules[__name__] = LazyImportModule(
+        __name__,
+        globals()['__file__'],
+        _import_structure,
+        module_spec=__spec__,
+        extra_objects={},
+    )

modelscope/models/nlp/polylm/text_generation.py

@@ -0,0 +1,55 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from collections import OrderedDict
+from typing import Dict, Generator
+
+import torch
+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
+
+from modelscope.metainfo import Models
+from modelscope.models.base import Tensor, TorchModel
+from modelscope.models.builder import MODELS
+from modelscope.utils.constant import Tasks
+from modelscope.utils.hub import read_config
+from modelscope.utils.streaming_output import StreamingOutputMixin
+
+__all__ = ['PolyLMForTextGeneration']
+
+
+@MODELS.register_module(Tasks.text_generation, module_name=Models.polylm)
+class PolyLMForTextGeneration(TorchModel, StreamingOutputMixin):
+
+    def __init__(self, model_dir: str, *args, **kwargs):
+        """initialize the text generation model from the `model_dir` path.
+
+        Args:
+            model_dir (str): the model path.
+        """
+        super().__init__(model_dir, *args, **kwargs)
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            model_dir, use_fast=False)
+        self.model = AutoModelForCausalLM.from_pretrained(
+            model_dir, device_map='auto')
+        self.model.eval()
+
+    def forward(self, input: Dict[str, Tensor], **kwargs) -> Dict[str, Tensor]:
+        """return the result by the model
+
+        Args:
+            input (Dict[str, Tensor]): the preprocessed data
+
+        Returns:
+            Dict[str, Tensor]: results
+        """
+        res = self.generate(input, **kwargs)
+        return res
+
+    def generate(self, input: Dict[str, Tensor],
+                 **kwargs) -> Dict[str, Tensor]:
+        device = self.model.device
+        inputs = self.tokenizer(input, return_tensors='pt')
+        outputs = self.model.generate(
+            inputs.input_ids.to(device),
+            attention_mask=inputs.attention_mask.to(device),
+            **kwargs)
+        pred = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
+        return pred

modelscope/pipelines/nlp/__init__.py

@@ -12,6 +12,7 @@ if TYPE_CHECKING:
     from .dialog_state_tracking_pipeline import DialogStateTrackingPipeline
     from .document_segmentation_pipeline import DocumentSegmentationPipeline
     from .extractive_summarization_pipeline import ExtractiveSummarizationPipeline
+    from .polylm_text_generation_pipeline import PolyLMTextGenerationPipeline
     from .fasttext_text_classification_pipeline import FasttextSequenceClassificationPipeline
     from .faq_question_answering_pipeline import FaqQuestionAnsweringPipeline
     from .feature_extraction_pipeline import FeatureExtractionPipeline
@@ -50,6 +51,7 @@ else:
         'automatic_post_editing_pipeline': ['AutomaticPostEditingPipeline'],
         'conversational_text_to_sql_pipeline':
         ['ConversationalTextToSqlPipeline'],
+        'polylm_text_generation_pipeline': ['PolyLMTextGenerationPipeline'],
         'dialog_intent_prediction_pipeline':
         ['DialogIntentPredictionPipeline'],
         'dialog_modeling_pipeline': ['DialogModelingPipeline'],

modelscope/pipelines/nlp/polylm_text_generation_pipeline.py

@@ -0,0 +1,66 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+# Copyright (c) 2022 Zhipu.AI
+import os
+from typing import Any, Dict, Union
+
+import torch
+
+from modelscope.metainfo import Pipelines
+from modelscope.models.base import Model
+from modelscope.models.nlp import PolyLMForTextGeneration
+from modelscope.outputs import OutputKeys
+from modelscope.pipelines.base import Pipeline
+from modelscope.pipelines.builder import PIPELINES
+from modelscope.utils.constant import Tasks
+from modelscope.utils.streaming_output import PipelineStreamingOutputMixin
+
+__all__ = ['PolyLMTextGenerationPipeline']
+
+
+@PIPELINES.register_module(
+    Tasks.text_generation, module_name=Pipelines.polylm_text_generation)
+class PolyLMTextGenerationPipeline(Pipeline, PipelineStreamingOutputMixin):
+    """ A polyglot large language for text generation pipeline.
+
+    Examples:
+
+    >>> from modelscope.pipelines import pipeline
+    >>> from modelscope.utils.constant import Tasks
+    >>> polylm_13b_model_id = 'damo/nlp_polylm_13b_text_generation'
+    >>> input_text = "Beijing is the capital of China.\nTranslate this sentence from English to Chinese."
+    >>> kwargs = {"do_sample": False, "num_beams": 4, "max_new_tokens": 128, "early_stopping": True, "eos_token_id": 2}
+    >>> pipeline_ins = pipeline(Tasks.text_generation, model=polylm_13b_model_id)
+    >>> result = pipeline_ins(input_text, **kwargs)
+    >>> print(result['text'])
+    >>>
+    """
+
+    def __init__(self, model: Union[Model, str], **kwargs):
+        """Use `model` and `preprocessor` to create a generation pipeline for prediction.
+
+        Args:
+            model (str or Model): Supply either a local model dir which supported the text generation task,
+            or a model id from the model hub, or a torch model instance.
+            preprocessor (Preprocessor): An optional preprocessor instance, please make sure the preprocessor fits for
+            the model if supplied.
+            kwargs (dict, `optional`):
+                Extra kwargs passed into the preprocessor's constructor.
+        """
+        model = PolyLMForTextGeneration(model, **kwargs) if isinstance(
+            model, str) else model
+        super().__init__(model=model, **kwargs)
+
+    def _sanitize_parameters(self, **pipeline_parameters):
+        return {}, pipeline_parameters, {}
+
+    def preprocess(self, input: str) -> Dict[str, Any]:
+        return input
+
+    def forward(self, inputs: Dict[str, Any],
+                **forward_params) -> Dict[str, Any]:
+        with torch.no_grad():
+            return self.model(inputs, **forward_params)
+
+    def postprocess(self, input, **kwargs) -> Dict[str, Any]:
+        return {OutputKeys.TEXT: input}

tests/pipelines/test_polylm_text_generation.py

@@ -0,0 +1,93 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import unittest
+
+from modelscope.hub.snapshot_download import snapshot_download
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+from modelscope.utils.test_utils import test_level
+
+
+class TextGenerationTest(unittest.TestCase):
+
+    def setUp(self) -> None:
+        self.polylm_13b_model_id = 'damo/nlp_polylm_13b_text_generation'
+        self.polylm_multialpaca_13b_model_id = 'damo/nlp_polylm_multialpaca_13b_text_generation'
+        self.polylm_multialpaca_ecomm_13b_model_id = 'damo/nlp_polylm_multialpaca_13b_text_generation_ecomm'
+        self.input_text = 'Beijing is the capital of China.\nTranslate this sentence from English to Chinese.'
+        self.ecomm_input_text = \
+            'Below is an instruction that describes the attribute value detection task. ' \
+            + 'Write a response that appropriately completes the request.\n' \
+            + '### Instruction:\n暗格格纹纹路搭配磨砂表面\n' \
+            + 'Extract all attribute value with attribute name about 鞋跟高度, 下摆类型, 工艺, 裙长, 腰型, 图案, 开衩类型, 风格, 领型, 版型, ' \
+            + '鞋帮高度, 裤长, 裤型, 适用季节, 厚度, 弹性, 形状, 开口深度, 靴筒高度, 颜色, 闭合方式, 材质, 袖长, 鞋头款式, 袖型, 口袋类型 in the sentence. \n' \
+            + '### Response:'
+
+    @unittest.skip('oom error for 13b model')
+    def test_polylm_13b_with_model_name(self):
+        kwargs = {
+            'do_sample': False,
+            'num_beams': 4,
+            'max_new_tokens': 128,
+            'early_stopping': True,
+            'eos_token_id': 2
+        }
+        pipeline_ins = pipeline(
+            Tasks.text_generation, model=self.polylm_13b_model_id)
+        result = pipeline_ins(self.input_text, **kwargs)
+        print(result['text'])
+
+    @unittest.skip('oom error for 13b model')
+    def test_polylm_multialpaca_13b_with_model_name(self):
+        kwargs = {
+            'do_sample': True,
+            'top_p': 0.8,
+            'temperature': 0.7,
+            'repetition_penalty': 1.02,
+            'max_new_tokens': 128,
+            'num_return_sequences': 1,
+            'early_stopping': True,
+            'eos_token_id': 2
+        }
+        pipeline_ins = pipeline(
+            Tasks.text_generation, model=self.polylm_multialpaca_13b_model_id)
+
+        input_text = f'{self.input_text}\n\n'
+        result = pipeline_ins(input_text, **kwargs)
+        print(result['text'])
+
+    @unittest.skip('oom error for 13b model')
+    def test_polylm_multialpaca_ecomm_13b_with_model_name(self):
+        kwargs = {
+            'do_sample': True,
+            'top_p': 0.9,
+            'temperature': 1.0,
+            'repetition_penalty': 1.02,
+            'max_new_tokens': 128,
+            'num_return_sequences': 1,
+            'early_stopping': True,
+            'eos_token_id': 2
+        }
+        pipeline_ins = pipeline(
+            Tasks.text_generation,
+            model=self.polylm_multialpaca_ecomm_13b_model_id)
+        input_text = f'{self.ecomm_input_text}\n\n'
+        result = pipeline_ins(input_text, **kwargs)
+        print(result['text'])
+
+    @unittest.skip('oom error for 13b model')
+    def test_run_with_default_model(self):
+        kwargs = {
+            'do_sample': False,
+            'num_beams': 4,
+            'max_new_tokens': 128,
+            'early_stopping': True,
+            'eos_token_id': 2
+        }
+        pipeline_ins = pipeline(
+            Tasks.text_generation, model=self.polylm_13b_model_id)
+        result = pipeline_ins(self.input_text, **kwargs)
+        print(result['text'])
+
+
+if __name__ == '__main__':
+    unittest.main()