Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2025-12-25 12:39:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix chatglm sp_tokenizer error

Browse Source 
Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/14352495
This commit is contained in:
wenmeng.zwm
2023-10-18 20:24:42 +08:00
committed by mulin.lyh
parent 6201d1bfbc
commit e10237074e
2 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
modelscope/models/nlp/chatglm/tokenization.py
View File

@@ -199,6 +199,10 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
padding_side='left',
num_image_tokens=20000,
**kwargs) -> None:
self.sp_tokenizer = SPTokenizer(
vocab_file, num_image_tokens=num_image_tokens)
super().__init__(
do_lower_case=do_lower_case,
remove_space=remove_space,
@@ -220,9 +224,6 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
self.end_token = end_token
self.mask_token = mask_token
self.gmask_token = gmask_token
self.sp_tokenizer = SPTokenizer(
vocab_file, num_image_tokens=num_image_tokens)
""" Initialisation """
@property

2
modelscope/models/nlp/llama/text_generation.py
View File

@@ -71,6 +71,8 @@ def get_chat_prompt(system: str, text: str, history: List[Tuple[str, str]],
# This file is mainly copied from the llama code of transformers
@MODELS.register_module(Tasks.chat, module_name=Models.llama2)
@MODELS.register_module(Tasks.chat, module_name=Models.llama)
@MODELS.register_module(Tasks.text_generation, module_name=Models.llama2)
@MODELS.register_module(Tasks.chat, module_name=Models.llama2)
@MODELS.register_module(Tasks.text_generation, module_name=Models.llama)