tastelikefeet
2025-01-06 17:29:44 +08:00
committed by suluyana
parent 49d80d9c1d
commit b5b08f9718

View File

@@ -416,6 +416,8 @@ class TokenClassificationTransformersPreprocessor(
offset_mapping = []
tokens = self.nlp_tokenizer.tokenizer.tokenize(text)
offset = 0
if getattr(self.nlp_tokenizer.tokenizer, 'do_lower_case', False):
text = text.lower()
for token in tokens:
is_start = (token[:2] != '##')
if is_start: