Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2025-12-25 12:39:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

[to #47818884] add default preprocessor for taskmodels

Browse Source
This commit is contained in:
zhangzhicheng.zzc
2023-02-14 11:50:40 +00:00
parent a0c020cf34
commit 57670f1e64
1 changed files with 22 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
modelscope/preprocessors/base.py
View File

@@ -4,7 +4,7 @@ from abc import ABC, abstractmethod
from copy import deepcopy
from typing import Any, Callable, Dict, Optional, Sequence, Union
from modelscope.metainfo import Models, Preprocessors
from modelscope.metainfo import Models, Preprocessors, TaskModels
from modelscope.utils.config import Config, ConfigDict
from modelscope.utils.constant import (DEFAULT_MODEL_REVISION, Invoke,
ModeKeys, Tasks)
@@ -127,7 +127,7 @@ PREPROCESSOR_MAP = {
(Models.veco, Tasks.sentence_similarity):
Preprocessors.sen_cls_tokenizer,
# taskmodels
# ner models
(Models.lcrf, Tasks.named_entity_recognition):
Preprocessors.sequence_labeling_tokenizer,
(Models.lcrf, Tasks.word_segmentation):
@@ -141,6 +141,26 @@ PREPROCESSOR_MAP = {
(Models.tcrf, Tasks.named_entity_recognition):
Preprocessors.sequence_labeling_tokenizer,
# task models
(TaskModels.token_classification, Tasks.token_classification):
Preprocessors.sequence_labeling_tokenizer,
(TaskModels.token_classification, Tasks.part_of_speech):
Preprocessors.sequence_labeling_tokenizer,
(TaskModels.token_classification, Tasks.named_entity_recognition):
Preprocessors.sequence_labeling_tokenizer,
(TaskModels.text_classification, Tasks.text_classification):
Preprocessors.sen_cls_tokenizer,
(TaskModels.fill_mask, Tasks.fill_mask):
Preprocessors.fill_mask,
(TaskModels.feature_extraction, Tasks.feature_extraction):
Preprocessors.feature_extraction,
(TaskModels.information_extraction, Tasks.information_extraction):
Preprocessors.re_tokenizer,
(TaskModels.text_ranking, Tasks.text_ranking):
Preprocessors.text_ranking,
(TaskModels.text_generation, Tasks.text_generation):
Preprocessors.text_gen_tokenizer,
# cv
(Models.tinynas_detection, Tasks.image_object_detection):
Preprocessors.object_detection_tinynas_preprocessor,