Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2025-12-25 12:39:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix chatglm2 can't find tokenizer issue

Browse Source 
Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/14335080
* fix chatglm2 can't find tokenizer issue
This commit is contained in:
mulin.lyh
2023-10-17 22:15:54 +08:00
parent 19e7c1c807
commit 66430171ae
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
modelscope/models/nlp/chatglm2/tokenization.py
View File

@@ -72,7 +72,6 @@ class ChatGLM2Tokenizer(PreTrainedTokenizer):
model_input_names = ['input_ids', 'attention_mask', 'position_ids']
def __init__(self, vocab_file, padding_side='left', **kwargs):
super().__init__(padding_side=padding_side, **kwargs)
self.name = 'GLMTokenizer'
self.vocab_file = vocab_file
@@ -82,6 +81,7 @@ class ChatGLM2Tokenizer(PreTrainedTokenizer):
'<eos>': self.tokenizer.eos_id,
'<pad>': self.tokenizer.pad_id
}
super().__init__(padding_side=padding_side, **kwargs)
def get_command(self, token):
if token in self.special_tokens: