mirror of
https://github.com/modelscope/modelscope.git
synced 2026-09-02 20:21:46 +02:00
5e469008fd
* add preprocessor module * add model base and builder * update task constant * add load image preprocessor and its dependency * add pipeline interface and UT covered * support default pipeline for task * add image matting pipeline * refine nlp tokenize interface * add nlp pipeline * fix UT failed * add test for Compose Link: https://code.aone.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/8769235 * add preprocessor module * add test for Compose * fix citest error * fix abs class error * add model base and builder * update task constant * add load image preprocessor and its dependency * add pipeline interface and UT covered * support default pipeline for task * refine models and pipeline interface * add pipeline folder structure * add image matting pipeline * refine nlp tokenize interface * add nlp pipeline 1.add preprossor model pipeline for nlp text classification 2. add corresponding test Link: https://code.aone.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/8757371 * new nlp pipeline * format pre-commit code * update easynlp pipeline * update model_name for easynlp pipeline; add test for maas_lib/utils/typeassert.py * update test_typeassert.py * refactor code 1. rename typeassert to type_assert 2. use lazy import to make easynlp dependency optional 3. refine image matting UT * fix linter test failed * update requirements.txt * fix UT failed * fix citest script to update requirements
55 lines
1.7 KiB
Python
55 lines
1.7 KiB
Python
# Copyright (c) Alibaba, Inc. and its affiliates.
|
|
|
|
import time
|
|
from collections.abc import Sequence
|
|
|
|
from .builder import PREPROCESSORS, build_preprocessor
|
|
|
|
|
|
@PREPROCESSORS.register_module()
|
|
class Compose(object):
|
|
"""Compose a data pipeline with a sequence of transforms.
|
|
Args:
|
|
transforms (list[dict | callable]):
|
|
Either config dicts of transforms or transform objects.
|
|
profiling (bool, optional): If set True, will profile and
|
|
print preprocess time for each step.
|
|
"""
|
|
|
|
def __init__(self, transforms, field_name=None, profiling=False):
|
|
assert isinstance(transforms, Sequence)
|
|
self.profiling = profiling
|
|
self.transforms = []
|
|
self.field_name = field_name
|
|
for transform in transforms:
|
|
if isinstance(transform, dict):
|
|
if self.field_name is None:
|
|
transform = build_preprocessor(transform, field_name)
|
|
self.transforms.append(transform)
|
|
elif callable(transform):
|
|
self.transforms.append(transform)
|
|
else:
|
|
raise TypeError('transform must be callable or a dict, but got'
|
|
f' {type(transform)}')
|
|
|
|
def __call__(self, data):
|
|
for t in self.transforms:
|
|
if self.profiling:
|
|
start = time.time()
|
|
|
|
data = t(data)
|
|
|
|
if self.profiling:
|
|
print(f'{t} time {time.time()-start}')
|
|
|
|
if data is None:
|
|
return None
|
|
return data
|
|
|
|
def __repr__(self):
|
|
format_string = self.__class__.__name__ + '('
|
|
for t in self.transforms:
|
|
format_string += f'\n {t}'
|
|
format_string += '\n)'
|
|
return format_string
|