mirror of
https://github.com/modelscope/modelscope.git
synced 2026-02-24 12:10:09 +01:00
fix https://www.modelscope.cn/models/iic/nlp_structbert_address-parsing_chinese_base/feedback/issueDetail/20431 (#1170)
This commit is contained in:
@@ -416,6 +416,8 @@ class TokenClassificationTransformersPreprocessor(
|
||||
offset_mapping = []
|
||||
tokens = self.nlp_tokenizer.tokenizer.tokenize(text)
|
||||
offset = 0
|
||||
if getattr(self.nlp_tokenizer.tokenizer, 'do_lower_case', False):
|
||||
text = text.lower()
|
||||
for token in tokens:
|
||||
is_start = (token[:2] != '##')
|
||||
if is_start:
|
||||
|
||||
Reference in New Issue
Block a user