Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2026-02-24 04:01:10 +01:00
Code Issues Packages Projects Releases Wiki Activity

[to #42322933] add use_fast for text_ranking

Browse Source
This commit is contained in:
zhangzhicheng.zzc
2023-02-15 07:29:46 +00:00
parent 422ebd1385
commit b99a8a94e6
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
modelscope/preprocessors/nlp/text_ranking_preprocessor.py
View File

@@ -49,6 +49,7 @@ class TextRankingTransformersPreprocessor(TextRankingPreprocessorBase):
max_length=None,
padding='max_length',
truncation=True,
use_fast=True,
**kwargs):
"""The tokenizer preprocessor class for the text ranking preprocessor.
@@ -70,7 +71,8 @@ class TextRankingTransformersPreprocessor(TextRankingPreprocessorBase):
self.tokenize_kwargs = kwargs
self.tokenize_kwargs['padding'] = padding
self.tokenize_kwargs['truncation'] = truncation
self.tokenizer = AutoTokenizer.from_pretrained(self.model_dir)
self.tokenizer = AutoTokenizer.from_pretrained(
self.model_dir, use_fast=use_fast)
@type_assert(object, dict)
def __call__(self, data: Dict, **kwargs) -> Dict[str, Any]: