Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2025-12-25 12:39:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

[to #42322933] update preprocessor for deberta v2

Browse Source 
相关改动已经在 model repo中进行了改动

Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/11714125

* update preprocessor for deberta
This commit is contained in:
zhangzhicheng.zzc
2023-02-21 22:41:49 +08:00
committed by wenmeng.zwm
parent 3e4ee5958a
commit fb9c41092a
1 changed files with 0 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
modelscope/preprocessors/nlp/transformers_tokenizer.py
View File

@@ -75,11 +75,6 @@ class NLPTokenizer:
# fast version lead to parallel inference failed
model_type = self.model_type
model_dir = self.model_dir
if model_type == Models.deberta_v2:
from modelscope.models.nlp.deberta_v2 import DebertaV2Tokenizer, DebertaV2TokenizerFast
tokenizer = DebertaV2TokenizerFast if self.use_fast else DebertaV2Tokenizer
return tokenizer.from_pretrained(
model_dir) if model_dir is not None else tokenizer()
if model_type in (Models.structbert, Models.gpt3, Models.palm,
Models.plug, Models.megatron_bert,