From b99a8a94e63afb678e3995bc48e31f47768cb835 Mon Sep 17 00:00:00 2001 From: "zhangzhicheng.zzc" Date: Wed, 15 Feb 2023 07:29:46 +0000 Subject: [PATCH] [to #42322933] add use_fast for text_ranking --- modelscope/preprocessors/nlp/text_ranking_preprocessor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modelscope/preprocessors/nlp/text_ranking_preprocessor.py b/modelscope/preprocessors/nlp/text_ranking_preprocessor.py index 2a2472a6..4ded00f1 100644 --- a/modelscope/preprocessors/nlp/text_ranking_preprocessor.py +++ b/modelscope/preprocessors/nlp/text_ranking_preprocessor.py @@ -49,6 +49,7 @@ class TextRankingTransformersPreprocessor(TextRankingPreprocessorBase): max_length=None, padding='max_length', truncation=True, + use_fast=True, **kwargs): """The tokenizer preprocessor class for the text ranking preprocessor. @@ -70,7 +71,8 @@ class TextRankingTransformersPreprocessor(TextRankingPreprocessorBase): self.tokenize_kwargs = kwargs self.tokenize_kwargs['padding'] = padding self.tokenize_kwargs['truncation'] = truncation - self.tokenizer = AutoTokenizer.from_pretrained(self.model_dir) + self.tokenizer = AutoTokenizer.from_pretrained( + self.model_dir, use_fast=use_fast) @type_assert(object, dict) def __call__(self, data: Dict, **kwargs) -> Dict[str, Any]: