mirror of
https://github.com/modelscope/modelscope.git
synced 2025-12-21 02:29:23 +01:00
d721fabb34
1.新增支持原始bert模型（非easynlp的 backbone prefix版本）
2.支持bert的在sequence classification/fill mask /token classification上的backbone head形式
3.统一了sequence classification几个任务的pipeline到一个类
4.fill mask 支持backbone head形式
5.token classification的几个子任务（ner，word seg， part of speech）的preprocessor 统一到了一起TokenClassificationPreprocessor
6. sequence classification的几个子任务（single classification， pair classification）的preprocessor 统一到了一起SequenceClassificationPreprocessor
7. 改动register中 cls的group_key 赋值位置，之前的group_key在多个decorators的情况下，会被覆盖，obj_cls的group_key信息不正确
8. 基于backbone head形式将 原本group_key和 module同名的情况尝试做调整，如下在modelscope/pipelines/nlp/sequence_classification_pipeline.py 中
原本
@PIPELINES.register_module(
Tasks.sentiment_classification, module_name=Pipelines.sentiment_classification)
改成
@PIPELINES.register_module(
Tasks.text_classification, module_name=Pipelines.sentiment_classification)
相应的configuration.json也有改动，这样的改动更符合任务和pipline（子任务）的关系。
8. 其他相应改动为支持上述功能
Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/10041463
114 lines
5.1 KiB
Python
114 lines
5.1 KiB
Python
# Copyright (c) Alibaba, Inc. and its affiliates.
|
|
|
|
import unittest
|
|
|
|
from modelscope.preprocessors import build_preprocessor, nlp
|
|
from modelscope.utils.constant import Fields, InputFields
|
|
from modelscope.utils.logger import get_logger
|
|
|
|
logger = get_logger()
|
|
|
|
|
|
class NLPPreprocessorTest(unittest.TestCase):
|
|
|
|
def test_tokenize(self):
|
|
cfg = dict(type='Tokenize', tokenizer_name='bert-base-cased')
|
|
preprocessor = build_preprocessor(cfg, Fields.nlp)
|
|
input = {
|
|
InputFields.text:
|
|
'Do not meddle in the affairs of wizards, '
|
|
'for they are subtle and quick to anger.'
|
|
}
|
|
output = preprocessor(input)
|
|
self.assertTrue(InputFields.text in output)
|
|
self.assertEqual(output['input_ids'], [
|
|
101, 2091, 1136, 1143, 13002, 1107, 1103, 5707, 1104, 16678, 1116,
|
|
117, 1111, 1152, 1132, 11515, 1105, 3613, 1106, 4470, 119, 102
|
|
])
|
|
self.assertEqual(
|
|
output['token_type_ids'],
|
|
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
|
|
self.assertEqual(
|
|
output['attention_mask'],
|
|
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])
|
|
|
|
def test_token_classification_tokenize(self):
|
|
with self.subTest(tokenizer_type='bert'):
|
|
cfg = dict(
|
|
type='token-cls-tokenizer',
|
|
model_dir='bert-base-cased',
|
|
label2id={
|
|
'O': 0,
|
|
'B': 1,
|
|
'I': 2
|
|
})
|
|
preprocessor = build_preprocessor(cfg, Fields.nlp)
|
|
input = 'Do not meddle in the affairs of wizards, ' \
|
|
'for they are subtle and quick to anger.'
|
|
output = preprocessor(input)
|
|
self.assertTrue(InputFields.text in output)
|
|
self.assertEqual(output['input_ids'].tolist()[0], [
|
|
101, 2091, 1136, 1143, 13002, 1107, 1103, 5707, 1104, 16678,
|
|
1116, 117, 1111, 1152, 1132, 11515, 1105, 3613, 1106, 4470,
|
|
119, 102
|
|
])
|
|
self.assertEqual(output['attention_mask'].tolist()[0], [
|
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
1
|
|
])
|
|
self.assertEqual(output['label_mask'].tolist()[0], [
|
|
False, True, True, True, False, True, True, True, True, True,
|
|
False, True, True, True, True, True, True, True, True, True,
|
|
True, False
|
|
])
|
|
self.assertEqual(output['offset_mapping'], [(0, 2), (3, 6),
|
|
(7, 13), (14, 16),
|
|
(17, 20), (21, 28),
|
|
(29, 31), (32, 39),
|
|
(39, 40), (41, 44),
|
|
(45, 49), (50, 53),
|
|
(54, 60), (61, 64),
|
|
(65, 70), (71, 73),
|
|
(74, 79), (79, 80)])
|
|
|
|
with self.subTest(tokenizer_type='roberta'):
|
|
cfg = dict(
|
|
type='token-cls-tokenizer',
|
|
model_dir='xlm-roberta-base',
|
|
label2id={
|
|
'O': 0,
|
|
'B': 1,
|
|
'I': 2
|
|
})
|
|
preprocessor = build_preprocessor(cfg, Fields.nlp)
|
|
input = 'Do not meddle in the affairs of wizards, ' \
|
|
'for they are subtle and quick to anger.'
|
|
output = preprocessor(input)
|
|
self.assertTrue(InputFields.text in output)
|
|
self.assertEqual(output['input_ids'].tolist()[0], [
|
|
0, 984, 959, 128, 19298, 23, 70, 103086, 7, 111, 6, 44239,
|
|
99397, 4, 100, 1836, 621, 1614, 17991, 136, 63773, 47, 348, 56,
|
|
5, 2
|
|
])
|
|
self.assertEqual(output['attention_mask'].tolist()[0], [
|
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
1, 1, 1, 1, 1
|
|
])
|
|
self.assertEqual(output['label_mask'].tolist()[0], [
|
|
False, True, True, True, False, True, True, True, False, True,
|
|
True, False, False, False, True, True, True, True, False, True,
|
|
True, True, True, False, False, False
|
|
])
|
|
self.assertEqual(output['offset_mapping'], [(0, 2), (3, 6),
|
|
(7, 13), (14, 16),
|
|
(17, 20), (21, 28),
|
|
(29, 31), (32, 40),
|
|
(41, 44), (45, 49),
|
|
(50, 53), (54, 60),
|
|
(61, 64), (65, 70),
|
|
(71, 73), (74, 80)])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|