From 45cf0035f414464cd8eabe099372893d72914e21 Mon Sep 17 00:00:00 2001 From: tastelikefeet <58414341+tastelikefeet@users.noreply.github.com> Date: Mon, 3 Jul 2023 23:16:38 +0800 Subject: [PATCH] fix chatglm2 evaluation error: hypothesis emtpy (#348) * fix evaluation error: hypothesis emtpy * fix pipeline * fix bug --- examples/pytorch/chatglm6b/chatglm_trainer.py | 6 ++++- examples/pytorch/chatglm6b/finetune.py | 14 +++++++---- .../chatglm6b/text_generation_metric.py | 2 +- .../pipelines/nlp/text_generation_pipeline.py | 23 ++++++++++++++----- 4 files changed, 33 insertions(+), 12 deletions(-) diff --git a/examples/pytorch/chatglm6b/chatglm_trainer.py b/examples/pytorch/chatglm6b/chatglm_trainer.py index b34563bd..efa4dfce 100644 --- a/examples/pytorch/chatglm6b/chatglm_trainer.py +++ b/examples/pytorch/chatglm6b/chatglm_trainer.py @@ -16,6 +16,8 @@ class Seq2SeqTrainer(EpochBasedTrainer): if ignore_pad_token_for_loss: tokens = np.where(tokens != -100, tokens, self.tokenizer.pad_token_id) + tokens = np.where(tokens < self.tokenizer.vocab_size, tokens, + self.tokenizer.pad_token_id) return [ t for t in self.tokenizer.batch_decode( tokens, skip_special_tokens=True) if t != '' @@ -59,7 +61,9 @@ class Seq2SeqTrainer(EpochBasedTrainer): gen_kwargs['input_ids'] = generation_inputs gen_kwargs['pad_token_id'] = self.tokenizer.pad_token_id - generated_tokens = self.model.generate(**gen_kwargs) + self.model.eval() + with torch.no_grad(): + generated_tokens = self.model.generate(**gen_kwargs) generated_tokens = generated_tokens[:, generation_inputs.size()[-1]:] # in case the batch is shorter than max length, the output should be padded diff --git a/examples/pytorch/chatglm6b/finetune.py b/examples/pytorch/chatglm6b/finetune.py index 3fa73ba0..bf3953e8 100644 --- a/examples/pytorch/chatglm6b/finetune.py +++ b/examples/pytorch/chatglm6b/finetune.py @@ -192,8 +192,15 @@ if config['model']['type'] == 'chatglm6b': model_config['model']['prefix_projection'] = args.prefix_projection tokenizer = ChatGLMTokenizer.from_pretrained(model_dir, trust_remote_code=True) + +device_map_kwargs = {} +device_kwargs = {} +if args.use_lora != 0: + device_kwargs['device_map'] = 'auto' + # No placement for model, leave the model to `device_map` + device_kwargs['device'] = 'cpu' model = Model.from_pretrained( - model_dir, cfg_dict=model_config, device_map='auto') + model_dir, cfg_dict=model_config, **device_map_kwargs) if args.ptuning_checkpoint is not None: # Evaluation @@ -378,8 +385,7 @@ trainer = Seq2SeqTrainer( seed=args.seed, data_collator=data_collator, remove_unused_data=True, - # No placement for model, leave the model to `device_map` - device='cpu', - cfg_modify_fn=cfg_modify_fn) + cfg_modify_fn=cfg_modify_fn, + **device_kwargs) trainer.tokenizer = tokenizer trainer.train() diff --git a/examples/pytorch/chatglm6b/text_generation_metric.py b/examples/pytorch/chatglm6b/text_generation_metric.py index 2083453a..536bbe06 100644 --- a/examples/pytorch/chatglm6b/text_generation_metric.py +++ b/examples/pytorch/chatglm6b/text_generation_metric.py @@ -53,7 +53,7 @@ class TextGenerationMetric(Metric): } for pred, label in zip(preds, labels): hypothesis = list(jieba.cut(pred)) - if len(hypothesis) == 0: + if len(hypothesis) == 0 or ''.join(hypothesis) == '.': hypothesis = [''] reference = list(jieba.cut(label)) rouge = Rouge() diff --git a/modelscope/pipelines/nlp/text_generation_pipeline.py b/modelscope/pipelines/nlp/text_generation_pipeline.py index a0e8a0ee..a7806702 100644 --- a/modelscope/pipelines/nlp/text_generation_pipeline.py +++ b/modelscope/pipelines/nlp/text_generation_pipeline.py @@ -6,6 +6,7 @@ from typing import Any, Dict, Optional, Union import torch +from modelscope import snapshot_download from modelscope.metainfo import Pipelines from modelscope.models.base import Model from modelscope.outputs import (ModelOutputBase, OutputKeys, @@ -192,9 +193,14 @@ class ChatGLM6bTextGenerationPipeline(Pipeline): quantization_bit=None, use_bf16=False, **kwargs): - from modelscope.models.nlp.chatglm.text_generation import ChatGLMForConditionalGeneration - model = ChatGLMForConditionalGeneration(model) if isinstance( - model, str) else model + from modelscope.models.nlp.chatglm.text_generation import ChatGLMForConditionalGeneration, ChatGLMConfig + if isinstance(model, str): + model_dir = snapshot_download( + model) if not os.path.exists(model) else model + config = ChatGLMConfig.from_pretrained(model_dir) + model = ChatGLMForConditionalGeneration(config).half() + if torch.cuda.is_available(): + model = model.cuda() if quantization_bit is not None: model = model.quantize(quantization_bit) if use_bf16: @@ -225,9 +231,14 @@ class ChatGLM6bV2TextGenerationPipeline(Pipeline): quantization_bit=None, use_bf16=False, **kwargs): - from modelscope.models.nlp import ChatGLM2ForConditionalGeneration, ChatGLM2Tokenizer - model = ChatGLM2ForConditionalGeneration(model) if isinstance( - model, str) else model + from modelscope.models.nlp import ChatGLM2ForConditionalGeneration, ChatGLM2Tokenizer, ChatGLM2Config + if isinstance(model, str): + model_dir = snapshot_download( + model) if not os.path.exists(model) else model + config = ChatGLM2Config.from_pretrained(model_dir) + model = ChatGLM2ForConditionalGeneration(config) + if torch.cuda.is_available(): + model = model.cuda() if quantization_bit is not None: model = model.quantize(quantization_bit) if use_bf16: