From fb9c41092a85751e1af93dad065de70f5cc89f03 Mon Sep 17 00:00:00 2001 From: "zhangzhicheng.zzc" Date: Tue, 21 Feb 2023 22:41:49 +0800 Subject: [PATCH] [to #42322933] update preprocessor for deberta v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 相关改动已经在 model repo中进行了改动 Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/11714125 * update preprocessor for deberta --- modelscope/preprocessors/nlp/transformers_tokenizer.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/modelscope/preprocessors/nlp/transformers_tokenizer.py b/modelscope/preprocessors/nlp/transformers_tokenizer.py index 7a4705b9..d03f2171 100644 --- a/modelscope/preprocessors/nlp/transformers_tokenizer.py +++ b/modelscope/preprocessors/nlp/transformers_tokenizer.py @@ -75,11 +75,6 @@ class NLPTokenizer: # fast version lead to parallel inference failed model_type = self.model_type model_dir = self.model_dir - if model_type == Models.deberta_v2: - from modelscope.models.nlp.deberta_v2 import DebertaV2Tokenizer, DebertaV2TokenizerFast - tokenizer = DebertaV2TokenizerFast if self.use_fast else DebertaV2Tokenizer - return tokenizer.from_pretrained( - model_dir) if model_dir is not None else tokenizer() if model_type in (Models.structbert, Models.gpt3, Models.palm, Models.plug, Models.megatron_bert,