From 672a4ba107d29490d8a3a93d00ae576e06296433 Mon Sep 17 00:00:00 2001 From: "yuze.zyz" Date: Mon, 9 Jan 2023 21:33:42 +0800 Subject: [PATCH] Refactor tinynas objectdetection & img-classification Refactor tinynas model & pipeline: 1. Move preprocess method out of model to image.py 2. Pipeline calls the model.__call__ method instead of inference method 3. Remove some obsolete code 4. Add a default preprocessor to preprocessor.py instead of change config in modelhub. 5. Standardize the return value of model Refactor general image classification pipeline: 1. Change the preprocessor build method of ofa to avoid dependencies between multi-modal and cv. 2. Move preprocess method out of pipeline to image.py Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/11185418 --- .../test/regression/tinynas_obj_detection.bin | 3 + .../vit_base_image_classification.bin | 3 + modelscope/metainfo.py | 2 + .../models/cv/tinynas_detection/detector.py | 51 ++---- modelscope/pipelines/base.py | 4 +- .../cv/image_classification_pipeline.py | 169 +++++++++--------- .../cv/tinynas_detection_pipeline.py | 64 ++++--- modelscope/preprocessors/base.py | 8 + modelscope/preprocessors/image.py | 106 +++++++++++ modelscope/utils/regress_test_utils.py | 23 ++- .../test_general_image_classification.py | 7 +- tests/pipelines/test_tinynas_detection.py | 9 +- 12 files changed, 285 insertions(+), 164 deletions(-) create mode 100644 data/test/regression/tinynas_obj_detection.bin create mode 100644 data/test/regression/vit_base_image_classification.bin diff --git a/data/test/regression/tinynas_obj_detection.bin b/data/test/regression/tinynas_obj_detection.bin new file mode 100644 index 00000000..1d958222 --- /dev/null +++ b/data/test/regression/tinynas_obj_detection.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:753728b02574958ac9018b235609b87fc99ee23d2dbbe579b98a9b12d7443cc4 +size 118048 diff --git a/data/test/regression/vit_base_image_classification.bin b/data/test/regression/vit_base_image_classification.bin new file mode 100644 index 00000000..768ddcc6 --- /dev/null +++ b/data/test/regression/vit_base_image_classification.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba58d77303a90ca0b971c9312928182c5f779465a0b12661be8b7c88bf2ff015 +size 44817 diff --git a/modelscope/metainfo.py b/modelscope/metainfo.py index 48c66b52..bda6f41b 100644 --- a/modelscope/metainfo.py +++ b/modelscope/metainfo.py @@ -421,6 +421,8 @@ class Preprocessors(object): # cv preprocessor load_image = 'load-image' + object_detection_tinynas_preprocessor = 'object-detection-tinynas-preprocessor' + image_classification_mmcv_preprocessor = 'image-classification-mmcv-preprocessor' image_denoie_preprocessor = 'image-denoise-preprocessor' image_color_enhance_preprocessor = 'image-color-enhance-preprocessor' image_instance_segmentation_preprocessor = 'image-instance-segmentation-preprocessor' diff --git a/modelscope/models/cv/tinynas_detection/detector.py b/modelscope/models/cv/tinynas_detection/detector.py index d7320aaa..b049f7a1 100644 --- a/modelscope/models/cv/tinynas_detection/detector.py +++ b/modelscope/models/cv/tinynas_detection/detector.py @@ -4,16 +4,12 @@ import os.path as osp import pickle -import cv2 import torch import torch.nn as nn import torchvision -from modelscope.metainfo import Models from modelscope.models.base.base_torch_model import TorchModel -from modelscope.models.builder import MODELS -from modelscope.utils.config import Config -from modelscope.utils.constant import ModelFile, Tasks +from modelscope.outputs.cv_outputs import DetectionOutput from .backbone import build_backbone from .head import build_head from .neck import build_neck @@ -67,41 +63,14 @@ class SingleStageDetector(TorchModel): m.eps = 1e-3 m.momentum = 0.03 - def inference(self, x): - + def forward(self, x): if self.training: - return self.forward_train(x) + pass else: - return self.forward_eval(x) - - def forward_train(self, x): - - pass - - def forward_eval(self, x): - - x = self.backbone(x) - x = self.neck(x) - prediction = self.head(x) - - return prediction - - def preprocess(self, image): - image = torch.from_numpy(image).type(torch.float32) - image = image.permute(2, 0, 1) - shape = image.shape # c, h, w - if self.size_divisible > 0: - import math - stride = self.size_divisible - shape = list(shape) - shape[1] = int(math.ceil(shape[1] / stride) * stride) - shape[2] = int(math.ceil(shape[2] / stride) * stride) - shape = tuple(shape) - pad_img = image.new(*shape).zero_() - pad_img[:, :image.shape[1], :image.shape[2]].copy_(image) - pad_img = pad_img.unsqueeze(0) - - return pad_img + x = self.backbone(x) + x = self.neck(x) + prediction = self.head(x) + return prediction def postprocess(self, preds): bboxes, scores, labels_idx = postprocess_gfocal( @@ -111,7 +80,11 @@ class SingleStageDetector(TorchModel): labels_idx = labels_idx.cpu().numpy() labels = [self.label_map[idx + 1][0]['name'] for idx in labels_idx] - return (bboxes, scores, labels) + return DetectionOutput( + boxes=bboxes, + scores=scores, + class_ids=labels, + ) def multiclass_nms(multi_bboxes, diff --git a/modelscope/pipelines/base.py b/modelscope/pipelines/base.py index d20549f2..a763018c 100644 --- a/modelscope/pipelines/base.py +++ b/modelscope/pipelines/base.py @@ -96,6 +96,7 @@ class Pipeline(ABC): if config_file is not None: self.cfg = Config.from_file(config_file) + model_dir = os.path.dirname(config_file) elif not self.has_multiple_models: if isinstance(self.model, str): model_dir = self.model @@ -103,8 +104,7 @@ class Pipeline(ABC): model_dir = self.model.model_dir self.cfg = read_config(model_dir) - if preprocessor is None and not self.has_multiple_models \ - and hasattr(self.cfg, 'preprocessor'): + if preprocessor is None and not self.has_multiple_models: self.preprocessor = Preprocessor.from_pretrained(model_dir) else: self.preprocessor = preprocessor diff --git a/modelscope/pipelines/cv/image_classification_pipeline.py b/modelscope/pipelines/cv/image_classification_pipeline.py index f5918dec..7dbe296a 100644 --- a/modelscope/pipelines/cv/image_classification_pipeline.py +++ b/modelscope/pipelines/cv/image_classification_pipeline.py @@ -1,21 +1,17 @@ # Copyright (c) Alibaba, Inc. and its affiliates. from typing import Any, Dict, Optional, Union -import cv2 import numpy as np -import PIL import torch -from modelscope.metainfo import Pipelines -from modelscope.models.multi_modal import OfaForAllTasks +from modelscope.metainfo import Pipelines, Preprocessors from modelscope.outputs import OutputKeys from modelscope.pipelines.base import Input, Model, Pipeline from modelscope.pipelines.builder import PIPELINES from modelscope.pipelines.util import batch_process -from modelscope.preprocessors import OfaPreprocessor, Preprocessor, load_image +from modelscope.preprocessors import Preprocessor from modelscope.preprocessors.image import LoadImage -from modelscope.utils.constant import Tasks -from modelscope.utils.device import get_device +from modelscope.utils.constant import Fields, Tasks from modelscope.utils.logger import get_logger logger = get_logger() @@ -23,35 +19,6 @@ logger = get_logger() @PIPELINES.register_module( Tasks.image_classification, module_name=Pipelines.image_classification) -class ImageClassificationPipeline(Pipeline): - - def __init__(self, - model: Union[Model, str], - preprocessor: Optional[Preprocessor] = None, - **kwargs): - super().__init__(model=model, preprocessor=preprocessor, **kwargs) - assert isinstance(model, str) or isinstance(model, Model), \ - 'model must be a single str or OfaForAllTasks' - self.model.eval() - self.model.to(get_device()) - if preprocessor is None and isinstance(self.model, OfaForAllTasks): - self.preprocessor = OfaPreprocessor(model_dir=self.model.model_dir) - - def _batch(self, data): - if isinstance(self.model, OfaForAllTasks): - return batch_process(self.model, data) - else: - return super(ImageClassificationPipeline, self)._batch(data) - - def forward(self, inputs: Dict[str, Any], - **forward_params) -> Dict[str, Any]: - with torch.no_grad(): - return super().forward(inputs, **forward_params) - - def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: - return inputs - - @PIPELINES.register_module( Tasks.image_classification, module_name=Pipelines.general_image_classification) @@ -69,68 +36,94 @@ class ImageClassificationPipeline(Pipeline): module_name=Pipelines.common_image_classification) class GeneralImageClassificationPipeline(Pipeline): - def __init__(self, model: str, **kwargs): - """ - use `model` and `preprocessor` to create a image classification pipeline for prediction + def __init__(self, + model: str, + preprocessor: Optional[Preprocessor] = None, + config_file: str = None, + device: str = 'gpu', + auto_collate=True, + **kwargs): + """Use `model` and `preprocessor` to create an image classification pipeline for prediction Args: - model: model id on modelscope hub. + model: A str format model id or model local dir to build the model instance from. + preprocessor: A preprocessor instance to preprocess the data, if None, + the pipeline will try to build the preprocessor according to the configuration.json file. + kwargs: The args needed by the `Pipeline` class. """ - super().__init__(model=model, **kwargs) - + super().__init__( + model=model, + preprocessor=preprocessor, + config_file=config_file, + device=device, + auto_collate=auto_collate) + self.target_gpus = None + if preprocessor is None: + assert hasattr(self.model, 'model_dir'), 'Model used in ImageClassificationPipeline should has ' \ + 'a `model_dir` attribute to build a preprocessor.' + if self.model.__class__.__name__ == 'OfaForAllTasks': + self.preprocessor = Preprocessor.from_pretrained( + model_name_or_path=self.model.model_dir, + type=Preprocessors.ofa_tasks_preprocessor, + field=Fields.multi_modal, + **kwargs) + else: + if next(self.model.parameters()).is_cuda: + self.target_gpus = [next(self.model.parameters()).device] + assert hasattr(self.model, 'model_dir'), 'Model used in GeneralImageClassificationPipeline' \ + ' should has a `model_dir` attribute to build a preprocessor.' + self.preprocessor = Preprocessor.from_pretrained( + self.model.model_dir, **kwargs) + if self.preprocessor.__class__.__name__ == 'ImageClassificationBypassPreprocessor': + from modelscope.preprocessors.image import ImageClassificationMmcvPreprocessor + self.preprocessor = ImageClassificationMmcvPreprocessor( + self.model.model_dir, **kwargs) logger.info('load model done') - def preprocess(self, input: Input) -> Dict[str, Any]: - from mmcls.datasets.pipelines import Compose - from mmcv.parallel import collate, scatter - from modelscope.models.cv.image_classification.utils import preprocess_transform - - img = LoadImage.convert_to_ndarray(input) # Default in RGB order - img = img[:, :, ::-1] # Convert to BGR - - cfg = self.model.cfg - - if self.model.config_type == 'mmcv_config': - if cfg.data.test.pipeline[0]['type'] == 'LoadImageFromFile': - cfg.data.test.pipeline.pop(0) - data = dict(img=img) - test_pipeline = Compose(cfg.data.test.pipeline) + def _batch(self, data): + if self.model.__class__.__name__ == 'OfaForAllTasks': + return batch_process(self.model, data) else: - if cfg.preprocessor.val[0]['type'] == 'LoadImageFromFile': - cfg.preprocessor.val.pop(0) - data = dict(img=img) - data_pipeline = preprocess_transform(cfg.preprocessor.val) - test_pipeline = Compose(data_pipeline) + return super()._batch(data) - data = test_pipeline(data) - data = collate([data], samples_per_gpu=1) - if next(self.model.parameters()).is_cuda: - # scatter to specified GPU - data = scatter(data, [next(self.model.parameters()).device])[0] + def preprocess(self, input: Input, **preprocess_params) -> Dict[str, Any]: + if self.model.__class__.__name__ == 'OfaForAllTasks': + return super().preprocess(input, **preprocess_params) + else: + img = LoadImage.convert_to_ndarray(input) + img = img[:, :, ::-1] # Convert to BGR + data = super().preprocess(img, **preprocess_params) + from mmcv.parallel import collate, scatter + data = collate([data], samples_per_gpu=1) + if self.target_gpus is not None: + # scatter to specified GPU + data = scatter(data, self.target_gpus)[0] + return data - return data - - def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: - - with torch.no_grad(): + def forward(self, input: Dict[str, Any], + **forward_params) -> Dict[str, Any]: + if self.model.__class__.__name__ != 'OfaForAllTasks': input['return_loss'] = False - scores = self.model(input) + return self.model(input) - return {'scores': scores} + def postprocess(self, inputs: Dict[str, Any], + **post_params) -> Dict[str, Any]: - def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + if self.model.__class__.__name__ != 'OfaForAllTasks': + scores = inputs - scores = inputs['scores'] + pred_scores = np.sort(scores, axis=1)[0][::-1][:5] + pred_labels = np.argsort(scores, axis=1)[0][::-1][:5] - pred_scores = np.sort(scores, axis=1)[0][::-1][:5] - pred_labels = np.argsort(scores, axis=1)[0][::-1][:5] + result = { + 'pred_score': [score for score in pred_scores], + 'pred_class': + [self.model.CLASSES[label] for label in pred_labels] + } - result = {'pred_score': [score for score in pred_scores]} - result['pred_class'] = [ - self.model.CLASSES[lable] for lable in pred_labels - ] - - outputs = { - OutputKeys.SCORES: result['pred_score'], - OutputKeys.LABELS: result['pred_class'] - } - return outputs + outputs = { + OutputKeys.SCORES: result['pred_score'], + OutputKeys.LABELS: result['pred_class'] + } + return outputs + else: + return inputs diff --git a/modelscope/pipelines/cv/tinynas_detection_pipeline.py b/modelscope/pipelines/cv/tinynas_detection_pipeline.py index 706582b8..c897af4d 100644 --- a/modelscope/pipelines/cv/tinynas_detection_pipeline.py +++ b/modelscope/pipelines/cv/tinynas_detection_pipeline.py @@ -1,16 +1,13 @@ # Copyright (c) Alibaba, Inc. and its affiliates. -from typing import Any, Dict - -import cv2 -import numpy as np -import torch +from typing import Any, Dict, Optional, Union from modelscope.metainfo import Pipelines from modelscope.outputs import OutputKeys +from modelscope.outputs.cv_outputs import DetectionOutput from modelscope.pipelines.base import Input, Pipeline from modelscope.pipelines.builder import PIPELINES -from modelscope.preprocessors import LoadImage +from modelscope.preprocessors import LoadImage, Preprocessor from modelscope.utils.constant import Tasks from modelscope.utils.cv.image_utils import \ show_image_object_detection_auto_result @@ -26,36 +23,47 @@ logger = get_logger() Tasks.image_object_detection, module_name=Pipelines.tinynas_detection) class TinynasDetectionPipeline(Pipeline): - def __init__(self, model: str, **kwargs): + def __init__(self, + model: str, + preprocessor: Optional[Preprocessor] = None, + **kwargs): + """Object detection pipeline, currently only for the tinynas-detection model. + + Args: + model: A str format model id or model local dir to build the model instance from. + preprocessor: A preprocessor instance to preprocess the data, if None, + the pipeline will try to build the preprocessor according to the configuration.json file. + kwargs: The args needed by the `Pipeline` class. """ - model: model id on modelscope hub. - """ - super().__init__(model=model, auto_collate=False, **kwargs) - if torch.cuda.is_available(): - self.device = 'cuda' - else: - self.device = 'cpu' - self.model.to(self.device) - self.model.eval() + super().__init__(model=model, preprocessor=preprocessor, **kwargs) def preprocess(self, input: Input) -> Dict[str, Any]: - img = LoadImage.convert_to_ndarray(input) - self.img = img - img = img.astype(np.float) - img = self.model.preprocess(img) - result = {'img': img.to(self.device)} - return result + return super().preprocess(img) - def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: + def forward( + self, input: Dict[str, + Any]) -> Union[Dict[str, Any], DetectionOutput]: + """The forward method of this pipeline. - outputs = self.model.inference(input['img']) - result = {'data': outputs} - return result + Args: + input: The input data output from the `preprocess` procedure. - def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + Returns: + A model output, either in a dict format, or in a standard `DetectionOutput` dataclass. + If outputs a dict, these keys are needed: + class_ids (`Tensor`, *optional*): class id for each object. + boxes (`Tensor`, *optional*): Bounding box for each detected object + in [left, top, right, bottom] format. + scores (`Tensor`, *optional*): Detection score for each object. + """ + return self.model(input['img']) - bboxes, scores, labels = self.model.postprocess(inputs['data']) + def postprocess( + self, inputs: Union[Dict[str, Any], + DetectionOutput]) -> Dict[str, Any]: + bboxes, scores, labels = inputs['boxes'], inputs['scores'], inputs[ + 'class_ids'] if bboxes is None: outputs = { OutputKeys.SCORES: [], diff --git a/modelscope/preprocessors/base.py b/modelscope/preprocessors/base.py index 4d25c4b2..d9b2836f 100644 --- a/modelscope/preprocessors/base.py +++ b/modelscope/preprocessors/base.py @@ -133,6 +133,14 @@ PREPROCESSOR_MAP = { Preprocessors.sequence_labeling_tokenizer, (Models.tcrf, Tasks.named_entity_recognition): Preprocessors.sequence_labeling_tokenizer, + + # cv + (Models.tinynas_detection, Tasks.image_object_detection): + Preprocessors.object_detection_tinynas_preprocessor, + (Models.tinynas_damoyolo, Tasks.image_object_detection): + Preprocessors.object_detection_tinynas_preprocessor, + (Models.tinynas_damoyolo, Tasks.domain_specific_object_detection): + Preprocessors.object_detection_tinynas_preprocessor, } diff --git a/modelscope/preprocessors/image.py b/modelscope/preprocessors/image.py index f0401f16..f93e51bd 100644 --- a/modelscope/preprocessors/image.py +++ b/modelscope/preprocessors/image.py @@ -1,5 +1,6 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import io +import os from typing import Any, Dict, Union import cv2 @@ -11,6 +12,7 @@ from PIL import Image, ImageOps from modelscope.fileio import File from modelscope.metainfo import Preprocessors from modelscope.utils.constant import Fields +from modelscope.utils.hub import read_config from modelscope.utils.type_assert import type_assert from .base import Preprocessor from .builder import PREPROCESSORS @@ -105,6 +107,110 @@ def load_image(image_path_or_url: str) -> Image.Image: return loader(image_path_or_url)['img'] +@PREPROCESSORS.register_module( + Fields.cv, module_name=Preprocessors.object_detection_tinynas_preprocessor) +class ObjectDetectionTinynasPreprocessor(Preprocessor): + + def __init__(self, size_divisible=32, **kwargs): + """Preprocess the image. + + What this preprocessor will do: + 1. Transpose the image matrix to make the channel the first dim. + 2. If the size_divisible is gt than 0, it will be used to pad the image. + 3. Expand an extra image dim as dim 0. + + Args: + size_divisible (int): The number will be used as a length unit to pad the image. + Formula: int(math.ceil(shape / size_divisible) * size_divisible) + Default 32. + """ + + super().__init__(**kwargs) + self.size_divisible = size_divisible + + @type_assert(object, object) + def __call__(self, data: np.ndarray) -> Dict[str, ndarray]: + """Preprocess the image. + + Args: + data: The input image with 3 dimensions. + + Returns: + The processed data in dict. + {'img': np.ndarray} + + """ + image = data.astype(np.float32) + image = image.transpose((2, 0, 1)) + shape = image.shape # c, h, w + if self.size_divisible > 0: + import math + stride = self.size_divisible + shape = list(shape) + shape[1] = int(math.ceil(shape[1] / stride) * stride) + shape[2] = int(math.ceil(shape[2] / stride) * stride) + shape = tuple(shape) + pad_img = np.zeros(shape).astype(np.float32) + pad_img[:, :image.shape[1], :image.shape[2]] = image + pad_img = np.expand_dims(pad_img, 0) + return {'img': pad_img} + + +@PREPROCESSORS.register_module( + Fields.cv, + module_name=Preprocessors.image_classification_mmcv_preprocessor) +class ImageClassificationMmcvPreprocessor(Preprocessor): + + def __init__(self, model_dir, **kwargs): + """Preprocess the image. + + What this preprocessor will do: + 1. Remove the `LoadImageFromFile` preprocessor(which will be called in the pipeline). + 2. Compose and instantiate other preprocessors configured in the file. + 3. Call the sub preprocessors one by one. + + This preprocessor supports two types of configuration: + 1. The mmcv config file, configured in a `config.py` + 2. The maas config file, configured in a `configuration.json` + By default, if the `config.py` exists, the preprocessor will use the mmcv config file. + + Args: + model_dir (str): The model dir to build the preprocessor from. + """ + + import mmcv + from mmcls.datasets.pipelines import Compose + from modelscope.models.cv.image_classification.utils import preprocess_transform + super().__init__(**kwargs) + + self.config_type = 'ms_config' + mm_config = os.path.join(model_dir, 'config.py') + if os.path.exists(mm_config): + cfg = mmcv.Config.fromfile(mm_config) + cfg.model.pretrained = None + config_type = 'mmcv_config' + else: + cfg = read_config(model_dir) + cfg.model.mm_model.pretrained = None + config_type = 'ms_config' + + if config_type == 'mmcv_config': + if cfg.data.test.pipeline[0]['type'] == 'LoadImageFromFile': + cfg.data.test.pipeline.pop(0) + self.preprocessors = Compose(cfg.data.test.pipeline) + else: + if cfg.preprocessor.val[0]['type'] == 'LoadImageFromFile': + cfg.preprocessor.val.pop(0) + data_pipeline = preprocess_transform(cfg.preprocessor.val) + self.preprocessors = Compose(data_pipeline) + + @type_assert(object, object) + def __call__(self, data: np.ndarray) -> Dict[str, ndarray]: + data = dict(img=data) + data = self.preprocessors(data) + return data + + @PREPROCESSORS.register_module( Fields.cv, module_name=Preprocessors.image_color_enhance_preprocessor) class ImageColorEnhanceFinetunePreprocessor(Preprocessor): diff --git a/modelscope/utils/regress_test_utils.py b/modelscope/utils/regress_test_utils.py index e7a47214..bae2edac 100644 --- a/modelscope/utils/regress_test_utils.py +++ b/modelscope/utils/regress_test_utils.py @@ -120,8 +120,19 @@ class RegressTool: with open(baseline, 'rb') as f: base = pickle.load(f) - print(f'baseline: {json.dumps(base, cls=NumpyEncoder)}') - print(f'latest : {json.dumps(io_json, cls=NumpyEncoder)}') + class SafeNumpyEncoder(NumpyEncoder): + + def default(self, obj): + try: + return super().default(obj) + except Exception: + print( + f'Type {obj.__class__} cannot be serialized and printed' + ) + return None + + print(f'baseline: {json.dumps(base, cls=SafeNumpyEncoder)}') + print(f'latest : {json.dumps(io_json, cls=SafeNumpyEncoder)}') if not compare_io_and_print(base, io_json, compare_fn, **kwargs): raise ValueError('Result not match!') @@ -519,7 +530,8 @@ def compare_arguments_nested(print_content, arg1, arg2, rtol=1.e-3, - atol=1.e-8): + atol=1.e-8, + ignore_unknown_type=True): type1 = type(arg1) type2 = type(arg2) if type1.__name__ != type2.__name__: @@ -594,7 +606,10 @@ def compare_arguments_nested(print_content, return False return True else: - raise ValueError(f'type not supported: {type1}') + if ignore_unknown_type: + return True + else: + raise ValueError(f'type not supported: {type1}') def compare_io_and_print(baseline_json, io_json, compare_fn=None, **kwargs): diff --git a/tests/pipelines/test_general_image_classification.py b/tests/pipelines/test_general_image_classification.py index 78440c69..6601f792 100644 --- a/tests/pipelines/test_general_image_classification.py +++ b/tests/pipelines/test_general_image_classification.py @@ -5,6 +5,7 @@ import unittest from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.demo_utils import DemoCompatibilityCheck +from modelscope.utils.regress_test_utils import MsRegressTool from modelscope.utils.test_utils import test_level @@ -14,6 +15,7 @@ class GeneralImageClassificationTest(unittest.TestCase, def setUp(self) -> None: self.task = Tasks.image_classification self.model_id = 'damo/cv_vit-base_image-classification_Dailylife-labels' + self.regress_tool = MsRegressTool(baseline=False) @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_run_ImageNet(self): @@ -28,7 +30,10 @@ class GeneralImageClassificationTest(unittest.TestCase, general_image_classification = pipeline( Tasks.image_classification, model='damo/cv_vit-base_image-classification_Dailylife-labels') - result = general_image_classification('data/test/images/bird.JPEG') + with self.regress_tool.monitor_module_single_forward( + general_image_classification.model, + 'vit_base_image_classification'): + result = general_image_classification('data/test/images/bird.JPEG') print(result) @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') diff --git a/tests/pipelines/test_tinynas_detection.py b/tests/pipelines/test_tinynas_detection.py index a73e7b0c..8cf6aef0 100644 --- a/tests/pipelines/test_tinynas_detection.py +++ b/tests/pipelines/test_tinynas_detection.py @@ -8,6 +8,7 @@ from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from modelscope.utils.demo_utils import DemoCompatibilityCheck +from modelscope.utils.regress_test_utils import MsRegressTool from modelscope.utils.test_utils import test_level @@ -16,13 +17,17 @@ class TinynasObjectDetectionTest(unittest.TestCase, DemoCompatibilityCheck): def setUp(self) -> None: self.task = Tasks.image_object_detection self.model_id = 'damo/cv_tinynas_object-detection_damoyolo' + self.regress_tool = MsRegressTool(baseline=False) @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_run_airdet(self): tinynas_object_detection = pipeline( Tasks.image_object_detection, model='damo/cv_tinynas_detection') - result = tinynas_object_detection( - 'data/test/images/image_detection.jpg') + with self.regress_tool.monitor_module_single_forward( + tinynas_object_detection.model, 'tinynas_obj_detection', + atol=1e-4): + result = tinynas_object_detection( + 'data/test/images/image_detection.jpg') print('airdet', result) @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')