Refactor tinynas objectdetection & img-classification

Refactor tinynas model & pipeline:
1. Move preprocess method out of model to image.py
2. Pipeline calls the model.__call__ method instead of inference method
3. Remove some obsolete code
4. Add a default preprocessor to preprocessor.py instead of change config in modelhub.
5. Standardize the return value of model

Refactor general image classification pipeline:
1. Change the preprocessor build method of ofa to avoid dependencies between multi-modal and cv.
2. Move preprocess method out of pipeline to image.py
        Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/11185418

data/test/regression/tinynas_obj_detection.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:753728b02574958ac9018b235609b87fc99ee23d2dbbe579b98a9b12d7443cc4
+size 118048

data/test/regression/vit_base_image_classification.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba58d77303a90ca0b971c9312928182c5f779465a0b12661be8b7c88bf2ff015
+size 44817

modelscope/metainfo.py

@@ -421,6 +421,8 @@ class Preprocessors(object):
 
     # cv preprocessor
     load_image = 'load-image'
+    object_detection_tinynas_preprocessor = 'object-detection-tinynas-preprocessor'
+    image_classification_mmcv_preprocessor = 'image-classification-mmcv-preprocessor'
     image_denoie_preprocessor = 'image-denoise-preprocessor'
     image_color_enhance_preprocessor = 'image-color-enhance-preprocessor'
     image_instance_segmentation_preprocessor = 'image-instance-segmentation-preprocessor'

modelscope/models/cv/tinynas_detection/detector.py

@@ -4,16 +4,12 @@
 import os.path as osp
 import pickle
 
-import cv2
 import torch
 import torch.nn as nn
 import torchvision
 
-from modelscope.metainfo import Models
 from modelscope.models.base.base_torch_model import TorchModel
-from modelscope.models.builder import MODELS
-from modelscope.utils.config import Config
-from modelscope.utils.constant import ModelFile, Tasks
+from modelscope.outputs.cv_outputs import DetectionOutput
 from .backbone import build_backbone
 from .head import build_head
 from .neck import build_neck
@@ -67,41 +63,14 @@ class SingleStageDetector(TorchModel):
                 m.eps = 1e-3
                 m.momentum = 0.03
 
-    def inference(self, x):
-
+    def forward(self, x):
         if self.training:
-            return self.forward_train(x)
+            pass
         else:
-            return self.forward_eval(x)
-
-    def forward_train(self, x):
-
-        pass
-
-    def forward_eval(self, x):
-
-        x = self.backbone(x)
-        x = self.neck(x)
-        prediction = self.head(x)
-
-        return prediction
-
-    def preprocess(self, image):
-        image = torch.from_numpy(image).type(torch.float32)
-        image = image.permute(2, 0, 1)
-        shape = image.shape  # c, h, w
-        if self.size_divisible > 0:
-            import math
-            stride = self.size_divisible
-            shape = list(shape)
-            shape[1] = int(math.ceil(shape[1] / stride) * stride)
-            shape[2] = int(math.ceil(shape[2] / stride) * stride)
-            shape = tuple(shape)
-        pad_img = image.new(*shape).zero_()
-        pad_img[:, :image.shape[1], :image.shape[2]].copy_(image)
-        pad_img = pad_img.unsqueeze(0)
-
-        return pad_img
+            x = self.backbone(x)
+            x = self.neck(x)
+            prediction = self.head(x)
+            return prediction
 
     def postprocess(self, preds):
         bboxes, scores, labels_idx = postprocess_gfocal(
@@ -111,7 +80,11 @@ class SingleStageDetector(TorchModel):
         labels_idx = labels_idx.cpu().numpy()
         labels = [self.label_map[idx + 1][0]['name'] for idx in labels_idx]
 
-        return (bboxes, scores, labels)
+        return DetectionOutput(
+            boxes=bboxes,
+            scores=scores,
+            class_ids=labels,
+        )
 
 
 def multiclass_nms(multi_bboxes,

modelscope/pipelines/base.py

@@ -96,6 +96,7 @@ class Pipeline(ABC):
 
         if config_file is not None:
             self.cfg = Config.from_file(config_file)
+            model_dir = os.path.dirname(config_file)
         elif not self.has_multiple_models:
             if isinstance(self.model, str):
                 model_dir = self.model
@@ -103,8 +104,7 @@ class Pipeline(ABC):
                 model_dir = self.model.model_dir
             self.cfg = read_config(model_dir)
 
-        if preprocessor is None and not self.has_multiple_models \
-           and hasattr(self.cfg, 'preprocessor'):
+        if preprocessor is None and not self.has_multiple_models:
             self.preprocessor = Preprocessor.from_pretrained(model_dir)
         else:
             self.preprocessor = preprocessor

modelscope/pipelines/cv/image_classification_pipeline.py

@@ -1,21 +1,17 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 from typing import Any, Dict, Optional, Union
 
-import cv2
 import numpy as np
-import PIL
 import torch
 
-from modelscope.metainfo import Pipelines
-from modelscope.models.multi_modal import OfaForAllTasks
+from modelscope.metainfo import Pipelines, Preprocessors
 from modelscope.outputs import OutputKeys
 from modelscope.pipelines.base import Input, Model, Pipeline
 from modelscope.pipelines.builder import PIPELINES
 from modelscope.pipelines.util import batch_process
-from modelscope.preprocessors import OfaPreprocessor, Preprocessor, load_image
+from modelscope.preprocessors import Preprocessor
 from modelscope.preprocessors.image import LoadImage
-from modelscope.utils.constant import Tasks
-from modelscope.utils.device import get_device
+from modelscope.utils.constant import Fields, Tasks
 from modelscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -23,35 +19,6 @@ logger = get_logger()
 
 @PIPELINES.register_module(
     Tasks.image_classification, module_name=Pipelines.image_classification)
-class ImageClassificationPipeline(Pipeline):
-
-    def __init__(self,
-                 model: Union[Model, str],
-                 preprocessor: Optional[Preprocessor] = None,
-                 **kwargs):
-        super().__init__(model=model, preprocessor=preprocessor, **kwargs)
-        assert isinstance(model, str) or isinstance(model, Model), \
-            'model must be a single str or OfaForAllTasks'
-        self.model.eval()
-        self.model.to(get_device())
-        if preprocessor is None and isinstance(self.model, OfaForAllTasks):
-            self.preprocessor = OfaPreprocessor(model_dir=self.model.model_dir)
-
-    def _batch(self, data):
-        if isinstance(self.model, OfaForAllTasks):
-            return batch_process(self.model, data)
-        else:
-            return super(ImageClassificationPipeline, self)._batch(data)
-
-    def forward(self, inputs: Dict[str, Any],
-                **forward_params) -> Dict[str, Any]:
-        with torch.no_grad():
-            return super().forward(inputs, **forward_params)
-
-    def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
-        return inputs
-
-
 @PIPELINES.register_module(
     Tasks.image_classification,
     module_name=Pipelines.general_image_classification)
@@ -69,68 +36,94 @@ class ImageClassificationPipeline(Pipeline):
     module_name=Pipelines.common_image_classification)
 class GeneralImageClassificationPipeline(Pipeline):
 
-    def __init__(self, model: str, **kwargs):
-        """
-        use `model` and `preprocessor` to create a image classification pipeline for prediction
+    def __init__(self,
+                 model: str,
+                 preprocessor: Optional[Preprocessor] = None,
+                 config_file: str = None,
+                 device: str = 'gpu',
+                 auto_collate=True,
+                 **kwargs):
+        """Use `model` and `preprocessor` to create an image classification pipeline for prediction
         Args:
-            model: model id on modelscope hub.
+            model: A str format model id or model local dir to build the model instance from.
+            preprocessor: A preprocessor instance to preprocess the data, if None,
+            the pipeline will try to build the preprocessor according to the configuration.json file.
+            kwargs: The args needed by the `Pipeline` class.
         """
-        super().__init__(model=model, **kwargs)
-
+        super().__init__(
+            model=model,
+            preprocessor=preprocessor,
+            config_file=config_file,
+            device=device,
+            auto_collate=auto_collate)
+        self.target_gpus = None
+        if preprocessor is None:
+            assert hasattr(self.model, 'model_dir'), 'Model used in ImageClassificationPipeline should has ' \
+                                                     'a `model_dir` attribute to build a preprocessor.'
+            if self.model.__class__.__name__ == 'OfaForAllTasks':
+                self.preprocessor = Preprocessor.from_pretrained(
+                    model_name_or_path=self.model.model_dir,
+                    type=Preprocessors.ofa_tasks_preprocessor,
+                    field=Fields.multi_modal,
+                    **kwargs)
+            else:
+                if next(self.model.parameters()).is_cuda:
+                    self.target_gpus = [next(self.model.parameters()).device]
+                assert hasattr(self.model, 'model_dir'), 'Model used in GeneralImageClassificationPipeline' \
+                                                         ' should has a `model_dir` attribute to build a preprocessor.'
+                self.preprocessor = Preprocessor.from_pretrained(
+                    self.model.model_dir, **kwargs)
+                if self.preprocessor.__class__.__name__ == 'ImageClassificationBypassPreprocessor':
+                    from modelscope.preprocessors.image import ImageClassificationMmcvPreprocessor
+                    self.preprocessor = ImageClassificationMmcvPreprocessor(
+                        self.model.model_dir, **kwargs)
         logger.info('load model done')
 
-    def preprocess(self, input: Input) -> Dict[str, Any]:
-        from mmcls.datasets.pipelines import Compose
-        from mmcv.parallel import collate, scatter
-        from modelscope.models.cv.image_classification.utils import preprocess_transform
-
-        img = LoadImage.convert_to_ndarray(input)  # Default in RGB order
-        img = img[:, :, ::-1]  # Convert to BGR
-
-        cfg = self.model.cfg
-
-        if self.model.config_type == 'mmcv_config':
-            if cfg.data.test.pipeline[0]['type'] == 'LoadImageFromFile':
-                cfg.data.test.pipeline.pop(0)
-            data = dict(img=img)
-            test_pipeline = Compose(cfg.data.test.pipeline)
+    def _batch(self, data):
+        if self.model.__class__.__name__ == 'OfaForAllTasks':
+            return batch_process(self.model, data)
         else:
-            if cfg.preprocessor.val[0]['type'] == 'LoadImageFromFile':
-                cfg.preprocessor.val.pop(0)
-            data = dict(img=img)
-            data_pipeline = preprocess_transform(cfg.preprocessor.val)
-            test_pipeline = Compose(data_pipeline)
+            return super()._batch(data)
 
-        data = test_pipeline(data)
-        data = collate([data], samples_per_gpu=1)
-        if next(self.model.parameters()).is_cuda:
-            # scatter to specified GPU
-            data = scatter(data, [next(self.model.parameters()).device])[0]
+    def preprocess(self, input: Input, **preprocess_params) -> Dict[str, Any]:
+        if self.model.__class__.__name__ == 'OfaForAllTasks':
+            return super().preprocess(input, **preprocess_params)
+        else:
+            img = LoadImage.convert_to_ndarray(input)
+            img = img[:, :, ::-1]  # Convert to BGR
+            data = super().preprocess(img, **preprocess_params)
+            from mmcv.parallel import collate, scatter
+            data = collate([data], samples_per_gpu=1)
+            if self.target_gpus is not None:
+                # scatter to specified GPU
+                data = scatter(data, self.target_gpus)[0]
+            return data
 
-        return data
-
-    def forward(self, input: Dict[str, Any]) -> Dict[str, Any]:
-
-        with torch.no_grad():
+    def forward(self, input: Dict[str, Any],
+                **forward_params) -> Dict[str, Any]:
+        if self.model.__class__.__name__ != 'OfaForAllTasks':
             input['return_loss'] = False
-            scores = self.model(input)
+        return self.model(input)
 
-        return {'scores': scores}
+    def postprocess(self, inputs: Dict[str, Any],
+                    **post_params) -> Dict[str, Any]:
 
-    def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        if self.model.__class__.__name__ != 'OfaForAllTasks':
+            scores = inputs
 
-        scores = inputs['scores']
+            pred_scores = np.sort(scores, axis=1)[0][::-1][:5]
+            pred_labels = np.argsort(scores, axis=1)[0][::-1][:5]
 
-        pred_scores = np.sort(scores, axis=1)[0][::-1][:5]
-        pred_labels = np.argsort(scores, axis=1)[0][::-1][:5]
+            result = {
+                'pred_score': [score for score in pred_scores],
+                'pred_class':
+                [self.model.CLASSES[label] for label in pred_labels]
+            }
 
-        result = {'pred_score': [score for score in pred_scores]}
-        result['pred_class'] = [
-            self.model.CLASSES[lable] for lable in pred_labels
-        ]
-
-        outputs = {
-            OutputKeys.SCORES: result['pred_score'],
-            OutputKeys.LABELS: result['pred_class']
-        }
-        return outputs
+            outputs = {
+                OutputKeys.SCORES: result['pred_score'],
+                OutputKeys.LABELS: result['pred_class']
+            }
+            return outputs
+        else:
+            return inputs

modelscope/pipelines/cv/tinynas_detection_pipeline.py

@@ -1,16 +1,13 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from typing import Any, Dict
-
-import cv2
-import numpy as np
-import torch
+from typing import Any, Dict, Optional, Union
 
 from modelscope.metainfo import Pipelines
 from modelscope.outputs import OutputKeys
+from modelscope.outputs.cv_outputs import DetectionOutput
 from modelscope.pipelines.base import Input, Pipeline
 from modelscope.pipelines.builder import PIPELINES
-from modelscope.preprocessors import LoadImage
+from modelscope.preprocessors import LoadImage, Preprocessor
 from modelscope.utils.constant import Tasks
 from modelscope.utils.cv.image_utils import \
     show_image_object_detection_auto_result
@@ -26,36 +23,47 @@ logger = get_logger()
     Tasks.image_object_detection, module_name=Pipelines.tinynas_detection)
 class TinynasDetectionPipeline(Pipeline):
 
-    def __init__(self, model: str, **kwargs):
+    def __init__(self,
+                 model: str,
+                 preprocessor: Optional[Preprocessor] = None,
+                 **kwargs):
+        """Object detection pipeline, currently only for the tinynas-detection model.
+
+        Args:
+            model: A str format model id or model local dir to build the model instance from.
+            preprocessor: A preprocessor instance to preprocess the data, if None,
+            the pipeline will try to build the preprocessor according to the configuration.json file.
+            kwargs: The args needed by the `Pipeline` class.
         """
-            model: model id on modelscope hub.
-        """
-        super().__init__(model=model, auto_collate=False, **kwargs)
-        if torch.cuda.is_available():
-            self.device = 'cuda'
-        else:
-            self.device = 'cpu'
-        self.model.to(self.device)
-        self.model.eval()
+        super().__init__(model=model, preprocessor=preprocessor, **kwargs)
 
     def preprocess(self, input: Input) -> Dict[str, Any]:
-
         img = LoadImage.convert_to_ndarray(input)
-        self.img = img
-        img = img.astype(np.float)
-        img = self.model.preprocess(img)
-        result = {'img': img.to(self.device)}
-        return result
+        return super().preprocess(img)
 
-    def forward(self, input: Dict[str, Any]) -> Dict[str, Any]:
+    def forward(
+            self, input: Dict[str,
+                              Any]) -> Union[Dict[str, Any], DetectionOutput]:
+        """The forward method of this pipeline.
 
-        outputs = self.model.inference(input['img'])
-        result = {'data': outputs}
-        return result
+        Args:
+            input: The input data output from the `preprocess` procedure.
 
-    def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        Returns:
+            A model output, either in a dict format, or in a standard `DetectionOutput` dataclass.
+            If outputs a dict, these keys are needed:
+                class_ids (`Tensor`, *optional*): class id for each object.
+                boxes (`Tensor`, *optional*): Bounding box for each detected object
+                    in [left, top, right, bottom] format.
+                scores (`Tensor`, *optional*): Detection score for each object.
+        """
+        return self.model(input['img'])
 
-        bboxes, scores, labels = self.model.postprocess(inputs['data'])
+    def postprocess(
+            self, inputs: Union[Dict[str, Any],
+                                DetectionOutput]) -> Dict[str, Any]:
+        bboxes, scores, labels = inputs['boxes'], inputs['scores'], inputs[
+            'class_ids']
         if bboxes is None:
             outputs = {
                 OutputKeys.SCORES: [],

modelscope/preprocessors/base.py

@@ -133,6 +133,14 @@ PREPROCESSOR_MAP = {
     Preprocessors.sequence_labeling_tokenizer,
     (Models.tcrf, Tasks.named_entity_recognition):
     Preprocessors.sequence_labeling_tokenizer,
+
+    # cv
+    (Models.tinynas_detection, Tasks.image_object_detection):
+    Preprocessors.object_detection_tinynas_preprocessor,
+    (Models.tinynas_damoyolo, Tasks.image_object_detection):
+    Preprocessors.object_detection_tinynas_preprocessor,
+    (Models.tinynas_damoyolo, Tasks.domain_specific_object_detection):
+    Preprocessors.object_detection_tinynas_preprocessor,
 }
 
 

modelscope/preprocessors/image.py

@@ -1,5 +1,6 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 import io
+import os
 from typing import Any, Dict, Union
 
 import cv2
@@ -11,6 +12,7 @@ from PIL import Image, ImageOps
 from modelscope.fileio import File
 from modelscope.metainfo import Preprocessors
 from modelscope.utils.constant import Fields
+from modelscope.utils.hub import read_config
 from modelscope.utils.type_assert import type_assert
 from .base import Preprocessor
 from .builder import PREPROCESSORS
@@ -105,6 +107,110 @@ def load_image(image_path_or_url: str) -> Image.Image:
     return loader(image_path_or_url)['img']
 
 
+@PREPROCESSORS.register_module(
+    Fields.cv, module_name=Preprocessors.object_detection_tinynas_preprocessor)
+class ObjectDetectionTinynasPreprocessor(Preprocessor):
+
+    def __init__(self, size_divisible=32, **kwargs):
+        """Preprocess the image.
+
+        What this preprocessor will do:
+        1. Transpose the image matrix to make the channel the first dim.
+        2. If the size_divisible is gt than 0, it will be used to pad the image.
+        3. Expand an extra image dim as dim 0.
+
+        Args:
+            size_divisible (int): The number will be used as a length unit to pad the image.
+                Formula: int(math.ceil(shape / size_divisible) * size_divisible)
+                Default 32.
+        """
+
+        super().__init__(**kwargs)
+        self.size_divisible = size_divisible
+
+    @type_assert(object, object)
+    def __call__(self, data: np.ndarray) -> Dict[str, ndarray]:
+        """Preprocess the image.
+
+        Args:
+            data: The input image with 3 dimensions.
+
+        Returns:
+            The processed data in dict.
+            {'img': np.ndarray}
+
+        """
+        image = data.astype(np.float32)
+        image = image.transpose((2, 0, 1))
+        shape = image.shape  # c, h, w
+        if self.size_divisible > 0:
+            import math
+            stride = self.size_divisible
+            shape = list(shape)
+            shape[1] = int(math.ceil(shape[1] / stride) * stride)
+            shape[2] = int(math.ceil(shape[2] / stride) * stride)
+            shape = tuple(shape)
+        pad_img = np.zeros(shape).astype(np.float32)
+        pad_img[:, :image.shape[1], :image.shape[2]] = image
+        pad_img = np.expand_dims(pad_img, 0)
+        return {'img': pad_img}
+
+
+@PREPROCESSORS.register_module(
+    Fields.cv,
+    module_name=Preprocessors.image_classification_mmcv_preprocessor)
+class ImageClassificationMmcvPreprocessor(Preprocessor):
+
+    def __init__(self, model_dir, **kwargs):
+        """Preprocess the image.
+
+        What this preprocessor will do:
+        1. Remove the `LoadImageFromFile` preprocessor(which will be called in the pipeline).
+        2. Compose and instantiate other preprocessors configured in the file.
+        3. Call the sub preprocessors one by one.
+
+        This preprocessor supports two types of configuration:
+        1. The mmcv config file, configured in a `config.py`
+        2. The maas config file, configured in a `configuration.json`
+        By default, if the `config.py` exists, the preprocessor will use the mmcv config file.
+
+        Args:
+            model_dir (str): The model dir to build the preprocessor from.
+        """
+
+        import mmcv
+        from mmcls.datasets.pipelines import Compose
+        from modelscope.models.cv.image_classification.utils import preprocess_transform
+        super().__init__(**kwargs)
+
+        self.config_type = 'ms_config'
+        mm_config = os.path.join(model_dir, 'config.py')
+        if os.path.exists(mm_config):
+            cfg = mmcv.Config.fromfile(mm_config)
+            cfg.model.pretrained = None
+            config_type = 'mmcv_config'
+        else:
+            cfg = read_config(model_dir)
+            cfg.model.mm_model.pretrained = None
+            config_type = 'ms_config'
+
+        if config_type == 'mmcv_config':
+            if cfg.data.test.pipeline[0]['type'] == 'LoadImageFromFile':
+                cfg.data.test.pipeline.pop(0)
+            self.preprocessors = Compose(cfg.data.test.pipeline)
+        else:
+            if cfg.preprocessor.val[0]['type'] == 'LoadImageFromFile':
+                cfg.preprocessor.val.pop(0)
+            data_pipeline = preprocess_transform(cfg.preprocessor.val)
+            self.preprocessors = Compose(data_pipeline)
+
+    @type_assert(object, object)
+    def __call__(self, data: np.ndarray) -> Dict[str, ndarray]:
+        data = dict(img=data)
+        data = self.preprocessors(data)
+        return data
+
+
 @PREPROCESSORS.register_module(
     Fields.cv, module_name=Preprocessors.image_color_enhance_preprocessor)
 class ImageColorEnhanceFinetunePreprocessor(Preprocessor):

modelscope/utils/regress_test_utils.py

@@ -120,8 +120,19 @@ class RegressTool:
             with open(baseline, 'rb') as f:
                 base = pickle.load(f)
 
-            print(f'baseline: {json.dumps(base, cls=NumpyEncoder)}')
-            print(f'latest  : {json.dumps(io_json, cls=NumpyEncoder)}')
+            class SafeNumpyEncoder(NumpyEncoder):
+
+                def default(self, obj):
+                    try:
+                        return super().default(obj)
+                    except Exception:
+                        print(
+                            f'Type {obj.__class__} cannot be serialized and printed'
+                        )
+                        return None
+
+            print(f'baseline: {json.dumps(base, cls=SafeNumpyEncoder)}')
+            print(f'latest  : {json.dumps(io_json, cls=SafeNumpyEncoder)}')
             if not compare_io_and_print(base, io_json, compare_fn, **kwargs):
                 raise ValueError('Result not match!')
 
@@ -519,7 +530,8 @@ def compare_arguments_nested(print_content,
                              arg1,
                              arg2,
                              rtol=1.e-3,
-                             atol=1.e-8):
+                             atol=1.e-8,
+                             ignore_unknown_type=True):
     type1 = type(arg1)
     type2 = type(arg2)
     if type1.__name__ != type2.__name__:
@@ -594,7 +606,10 @@ def compare_arguments_nested(print_content,
             return False
         return True
     else:
-        raise ValueError(f'type not supported: {type1}')
+        if ignore_unknown_type:
+            return True
+        else:
+            raise ValueError(f'type not supported: {type1}')
 
 
 def compare_io_and_print(baseline_json, io_json, compare_fn=None, **kwargs):

tests/pipelines/test_general_image_classification.py

@@ -5,6 +5,7 @@ import unittest
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
 from modelscope.utils.demo_utils import DemoCompatibilityCheck
+from modelscope.utils.regress_test_utils import MsRegressTool
 from modelscope.utils.test_utils import test_level
 
 
@@ -14,6 +15,7 @@ class GeneralImageClassificationTest(unittest.TestCase,
     def setUp(self) -> None:
         self.task = Tasks.image_classification
         self.model_id = 'damo/cv_vit-base_image-classification_Dailylife-labels'
+        self.regress_tool = MsRegressTool(baseline=False)
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_run_ImageNet(self):
@@ -28,7 +30,10 @@ class GeneralImageClassificationTest(unittest.TestCase,
         general_image_classification = pipeline(
             Tasks.image_classification,
             model='damo/cv_vit-base_image-classification_Dailylife-labels')
-        result = general_image_classification('data/test/images/bird.JPEG')
+        with self.regress_tool.monitor_module_single_forward(
+                general_image_classification.model,
+                'vit_base_image_classification'):
+            result = general_image_classification('data/test/images/bird.JPEG')
         print(result)
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')

tests/pipelines/test_tinynas_detection.py

@@ -8,6 +8,7 @@ from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
 from modelscope.utils.constant import Tasks
 from modelscope.utils.demo_utils import DemoCompatibilityCheck
+from modelscope.utils.regress_test_utils import MsRegressTool
 from modelscope.utils.test_utils import test_level
 
 
@@ -16,13 +17,17 @@ class TinynasObjectDetectionTest(unittest.TestCase, DemoCompatibilityCheck):
     def setUp(self) -> None:
         self.task = Tasks.image_object_detection
         self.model_id = 'damo/cv_tinynas_object-detection_damoyolo'
+        self.regress_tool = MsRegressTool(baseline=False)
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_run_airdet(self):
         tinynas_object_detection = pipeline(
             Tasks.image_object_detection, model='damo/cv_tinynas_detection')
-        result = tinynas_object_detection(
-            'data/test/images/image_detection.jpg')
+        with self.regress_tool.monitor_module_single_forward(
+                tinynas_object_detection.model, 'tinynas_obj_detection',
+                atol=1e-4):
+            result = tinynas_object_detection(
+                'data/test/images/image_detection.jpg')
         print('airdet', result)
 
     @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')