change use_fast_att and fix bfloat loading

Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/13522792

* check flash att is installed even if  use_fast_att is set True and fix bfloat loading

* skip pipeline model placement for quantization

* update unittest for qwen

modelscope/models/nlp/qwen/backbone.py

@@ -164,7 +164,7 @@ class QWenAttention(nn.Module):
         self.c_proj = nn.Linear(
             config.hidden_size, self.projection_size, bias=not config.no_bias)
 
-        if self.use_flash_attn:
+        if self.use_flash_attn and flash_attn_unpadded_func is not None:
             self.core_attention_flash = FlashSelfAttention(
                 causal=True, attention_dropout=config.attn_pdrop)
 
@@ -329,7 +329,7 @@ class QWenAttention(nn.Module):
         else:
             present = None
 
-        if self.use_flash_attn:
+        if self.use_flash_attn and flash_attn_unpadded_func is not None:
             q, k, v = query, key, value
             context_layer = self.core_attention_flash(q, k, v)
 
@@ -347,7 +347,7 @@ class QWenAttention(nn.Module):
         attn_output = self.c_proj(context_layer)
         outputs = (attn_output, present)
         if output_attentions:
-            if self.use_flash_attn:
+            if self.use_flash_attn and flash_attn_unpadded_func is not None:
                 raise ValueError(
                     'Cannot output attentions while using flash-attn')
             else:
@@ -713,8 +713,7 @@ class RotaryEmbedding(torch.nn.Module):
     def __init__(self, dim, base=10000, ntk_alpha=1.0):
         super().__init__()
         base = base * ntk_alpha**(dim / (dim - 2))
-        inv_freq = 1.0 / (base**(torch.arange(0, dim, 2).float() / dim))
-        self.register_buffer('inv_freq', inv_freq)
+        self.inv_freq = 1.0 / (base**(torch.arange(0, dim, 2).float() / dim))
         if importlib.util.find_spec('einops') is None:
             raise RuntimeError('einops is required for Rotary Embedding')
 

modelscope/models/nlp/qwen/text_generation.py

@@ -45,6 +45,14 @@ class QWenForTextGeneration(QWenPreTrainedModel):
         super().__init__(config)
         self.transformer = QWenModel(config)
         self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
+        assert not (config.bf16 and config.fp16), (
+            'In config, bf16 and fp16 cannot both be true')
+        if config.bf16:
+            self.transformer.bfloat16()
+            self.lm_head.bfloat16()
+        if config.fp16:
+            self.transformer.half()
+            self.lm_head.half()
         self.post_init()
 
     def get_output_embeddings(self):
@@ -242,5 +250,4 @@ class QWenForTextGeneration(QWenPreTrainedModel):
             prefix_allowed_tokens_fn,
             synced_gpus,
             streamer=streamer,
-            **kwargs,
-        )
+            **kwargs)

modelscope/pipelines/nlp/text_generation_pipeline.py

@@ -320,6 +320,8 @@ class QWenChatPipeline(Pipeline):
         self.tokenizer = QWenTokenizer.from_pretrained(self.model.model_dir)
 
         super().__init__(model=model, **kwargs)
+        # skip pipeline model placement
+        self._model_prepare = True
 
     def _sanitize_parameters(self, **pipeline_parameters):
         return {}, pipeline_parameters, {}
@@ -383,6 +385,8 @@ class QWenTextGenerationPipeline(Pipeline):
         self.tokenizer = QWenTokenizer.from_pretrained(self.model.model_dir)
 
         super().__init__(model=model, **kwargs)
+        # skip pipeline model placement
+        self._model_prepare = True
 
     def _sanitize_parameters(self, **pipeline_parameters):
         return {}, pipeline_parameters, {}

tests/pipelines/test_qwen_text_generation_pipeline.py

@@ -13,8 +13,8 @@ from modelscope.utils.test_utils import test_level
 class QWenTextGenerationPipelineTest(unittest.TestCase):
 
     def setUp(self) -> None:
-        self.qwen_base = '../qwen_7b_ckpt_modelscope/'  # local test only
-        self.qwen_chat = '../qwen_7b_ckpt_chat_modelscope/'  # local test only
+        self.qwen_base = 'qwen/Qwen-7B'
+        self.qwen_chat = 'qwen/Qwen-7B-Chat'
 
         self.qwen_base_input = '蒙古国的首都是乌兰巴托（Ulaanbaatar）\n冰岛的首都是雷克雅未克（Reykjavik）\n埃塞俄比亚的首都是'
         self.qwen_chat_input = [
@@ -54,7 +54,7 @@ class QWenTextGenerationPipelineTest(unittest.TestCase):
             print('Response:', response, end='\n')
 
     # 7B_ms_base
-    @unittest.skipUnless(test_level() >= 3, 'skip test in current test level')
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_qwen_base_with_text_generation(self):
         self.run_pipeline_with_model_id(
             self.qwen_base,
@@ -64,7 +64,7 @@ class QWenTextGenerationPipelineTest(unittest.TestCase):
             })
 
     # 7B_ms_base
-    @unittest.skipUnless(test_level() >= 3, 'skip test in current test level')
+    @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
     def test_qwen_base_with_text_generation_quant_int8(self):
         quantization_config = BitsAndBytesConfig(load_in_8bit=True)
 
@@ -78,7 +78,7 @@ class QWenTextGenerationPipelineTest(unittest.TestCase):
             })
 
     # 7B_ms_base
-    @unittest.skipUnless(test_level() >= 3, 'skip test in current test level')
+    @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
     def test_qwen_base_with_text_generation_quant_int4(self):
         quantization_config = BitsAndBytesConfig(
             load_in_4bit=True,
@@ -95,7 +95,7 @@ class QWenTextGenerationPipelineTest(unittest.TestCase):
             })
 
     # 7B_ms_chat
-    @unittest.skipUnless(test_level() >= 3, 'skip test in current test level')
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_qwen_chat_with_chat(self):
         self.run_chat_pipeline_with_model_id(
             self.qwen_chat,
@@ -105,7 +105,7 @@ class QWenTextGenerationPipelineTest(unittest.TestCase):
             })
 
     # 7B_ms_chat
-    @unittest.skipUnless(test_level() >= 3, 'skip test in current test level')
+    @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
     def test_qwen_chat_with_chat_quant_int8(self):
         quantization_config = BitsAndBytesConfig(load_in_8bit=True)
 
@@ -119,7 +119,7 @@ class QWenTextGenerationPipelineTest(unittest.TestCase):
             })
 
     # 7B_ms_base
-    @unittest.skipUnless(test_level() >= 3, 'skip test in current test level')
+    @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
     def test_qwen_chat_with_chat_quant_int4(self):
         quantization_config = BitsAndBytesConfig(
             load_in_4bit=True,