multi-modal/gridvlp: add multi-modal_gridvlp_classification_chinese-base-ecom-cate

Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/11436832

modelscope/metainfo.py

@@ -416,6 +416,8 @@ class Pipelines(object):
     diffusers_stable_diffusion = 'diffusers-stable-diffusion'
     document_vl_embedding = 'document-vl-embedding'
     chinese_stable_diffusion = 'chinese-stable-diffusion'
+    gridvlp_multi_modal_classification = 'gridvlp-multi-modal-classification'
+    gridvlp_multi_modal_embedding = 'gridvlp-multi-modal-embedding'
 
     # science tasks
     protein_structure = 'unifold-protein-structure'

modelscope/pipelines/multi_modal/gridvlp_pipeline.py

@@ -0,0 +1,276 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import os.path as osp
+import time
+import traceback
+from typing import Any, Dict, Optional
+
+import json
+import numpy as np
+import torch
+from PIL import Image
+from transformers import BertTokenizer
+
+from modelscope.hub.snapshot_download import snapshot_download
+from modelscope.metainfo import Pipelines
+from modelscope.pipelines import Pipeline
+from modelscope.pipelines.builder import PIPELINES
+from modelscope.preprocessors.image import load_image
+from modelscope.utils.constant import (DEFAULT_MODEL_REVISION, Frameworks,
+                                       Invoke, Tasks)
+from modelscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+def cost(end, begin):
+    return '{:.2f}ms'.format((end - begin) * 1000)
+
+
+class Config:
+    SCALE = 1 / 255.0
+    MEAN = np.require([0.485, 0.456, 0.406], dtype=np.float32)[:, np.newaxis,
+                                                               np.newaxis]
+    STD = np.require([0.229, 0.224, 0.225], dtype=np.float32)[:, np.newaxis,
+                                                              np.newaxis]
+
+    # RESIZE_HEIGHT = int(224*1.14)
+    RESIZE_HEIGHT = int(256)
+    # RESIZE_WIDTH = int(224*1.14)
+    RESIZE_WIDTH = int(256)
+    CROP_SIZE = 224
+
+
+def pre_processor(img):
+    img = img.convert('RGB')
+
+    w, h = img.size
+    if (w <= h and w == Config.RESIZE_WIDTH) \
+            or (h <= w and h == Config.RESIZE_WIDTH):
+        img = img
+    if w < h:
+        ow = Config.RESIZE_WIDTH
+        oh = int(Config.RESIZE_WIDTH * h / w)
+        img = img.resize((ow, oh), Image.BILINEAR)
+    else:
+        oh = Config.RESIZE_WIDTH
+        ow = int(Config.RESIZE_WIDTH * w / h)
+        img = img.resize((ow, oh), Image.BILINEAR)
+    w, h = img.size
+    crop_top = int(round((h - Config.CROP_SIZE) / 2.))
+    crop_left = int(round((w - Config.CROP_SIZE) / 2.))
+    img = img.crop((crop_left, crop_top, crop_left + Config.CROP_SIZE,
+                    crop_top + Config.CROP_SIZE))
+    _img = np.array(img, dtype=np.float32)
+    _img = np.require(_img.transpose((2, 0, 1)), dtype=np.float32)
+    _img *= Config.SCALE
+    _img -= Config.MEAN
+    _img /= Config.STD
+    return _img
+
+
+class GridVlpPipeline(Pipeline):
+    """ Pipeline for gridvlp, including classification and embedding."""
+
+    def __init__(self, model_name_or_path: str, **kwargs):
+        """ Pipeline for gridvlp, including classification and embedding.
+        Args:
+            model: path to local model directory.
+        """
+        # download model from modelscope to local model dir
+        logger.info(f'load checkpoint from modelscope {model_name_or_path}')
+        if osp.exists(model_name_or_path):
+            local_model_dir = model_name_or_path
+        else:
+            invoked_by = '%s/%s' % (Invoke.KEY, Invoke.PIPELINE)
+            local_model_dir = snapshot_download(
+                model_name_or_path,
+                DEFAULT_MODEL_REVISION,
+                user_agent=invoked_by)
+        self.local_model_dir = local_model_dir
+
+        # load model from cpu and torch jit model
+        logger.info(f'load model from {local_model_dir}')
+        self.model = torch.jit.load(
+            osp.join(local_model_dir, 'pytorch_model.pt'))
+        self.framework = Frameworks.torch
+        self.device_name = 'cpu'
+        self._model_prepare = True
+        self._auto_collate = False
+
+        # load tokenizer
+        logger.info(f'load tokenizer from {local_model_dir}')
+        self.tokenizer = BertTokenizer.from_pretrained(local_model_dir)
+
+    def preprocess(self, inputs: Dict[str, Any], max_seq_length=49):
+        # fetch input params
+        image = inputs.get('image', '')
+        text = inputs.get('text', '')
+
+        s1 = time.time()
+
+        # download image and preprocess
+        try:
+            # load PIL image
+            img = load_image(image)
+            s2 = time.time()
+
+            # image preprocess
+            image_data = pre_processor(img)
+            s3 = time.time()
+
+        except Exception:
+            image_data = np.zeros((3, 224, 224), dtype=np.float32)
+            s2 = time.time()
+            s3 = time.time()
+            logger.info(traceback.print_exc())
+
+        # text process
+        if text is None or text.isspace() or not text.strip():
+            logger.info('text is empty!')
+            text = ''
+        inputs = self.tokenizer(
+            text,
+            padding='max_length',
+            truncation=True,
+            max_length=max_seq_length)
+
+        s4 = time.time()
+
+        logger.info(f'example. text: {text} image: {image}')
+        logger.info(
+            f'preprocess. Img_Download:{cost(s2, s1)}, Img_Pre:{cost(s3, s2)}, Txt_Pre:{cost(s4, s3)}'
+        )
+
+        input_dict = {
+            'image': image_data,
+            'input_ids': inputs['input_ids'],
+            'input_mask': inputs['attention_mask'],
+            'segment_ids': inputs['token_type_ids']
+        }
+        return input_dict
+
+
+@PIPELINES.register_module(
+    Tasks.visual_question_answering,
+    module_name=Pipelines.gridvlp_multi_modal_classification)
+class GridVlpClassificationPipeline(GridVlpPipeline):
+    """ Pipeline for gridvlp classification, including cate classfication and
+    brand classification.
+
+    Example:
+
+    ```python
+    >>> from modelscope.pipelines.multi_modal.gridvlp_pipeline import \
+    GridVlpClassificationPipeline
+
+    >>> pipeline = GridVlpClassificationPipeline('rgtjf1/multi-modal_gridvlp_classification_chinese-base-ecom-cate')
+    >>> output = pipeline({'text': '女装快干弹力轻型短裤448575',\
+        'image':'https://yejiabo-public.oss-cn-zhangjiakou.aliyuncs.com/alinlp/clothes.png'})
+    >>> output['text'][0]
+    {'label': {'cate_name': '休闲裤', 'cate_path': '女装>>裤子>>休闲裤>>休闲裤'}, 'score': 0.4146, 'rank': 0}
+
+    ```
+    """
+
+    def __init__(self, model_name_or_path: str, **kwargs):
+        """ Pipeline for gridvlp classification, including cate classfication and
+    brand classification.
+        Args:
+            model: path to local model directory.
+        """
+        super().__init__(model_name_or_path, **kwargs)
+
+        # load label mapping
+        logger.info(f'load label mapping from {self.local_model_dir}')
+        self.label_mapping = json.load(
+            open(osp.join(self.local_model_dir, 'label_mapping.json')))
+
+    def forward(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        s4 = time.time()
+
+        box_tensor = torch.zeros(1, dtype=torch.float32)
+
+        output = self.model(
+            torch.tensor(inputs['image']).unsqueeze(0),
+            box_tensor.unsqueeze(0),
+            torch.tensor(inputs['input_ids'], dtype=torch.long).unsqueeze(0),
+            torch.tensor(inputs['input_mask'], dtype=torch.long).unsqueeze(0),
+            torch.tensor(inputs['segment_ids'], dtype=torch.long).unsqueeze(0))
+        output = output[0].detach().numpy()
+
+        s5 = time.time()
+
+        logger.info(f'forward. Infer:{cost(s5, s4)}')
+
+        # 返回结果
+        return output
+
+    def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        s5 = time.time()
+        output = inputs
+        index = np.argsort(-output)
+        out_sort = output[index]
+
+        top_k = []
+        for i in range(min(10, len(self.label_mapping))):
+            label = self.label_mapping[str(index[i])]
+            top_k.append({
+                'label': label,
+                'score': round(float(out_sort[i]), 4),
+                'rank': i
+            })
+
+        s6 = time.time()
+        logger.info(f'postprocess. Post: {cost(s6, s5)}')
+        return {'text': top_k}
+
+
+@PIPELINES.register_module(
+    Tasks.multi_modal_embedding,
+    module_name=Pipelines.gridvlp_multi_modal_embedding)
+class GridVlpEmbeddingPipeline(GridVlpPipeline):
+    """ Pipeline for gridvlp embedding. These only generate unified multi-modal
+    embeddings and output it in `text_embedding` or `img_embedding`.
+
+    Example:
+
+    ```python
+    >>> from modelscope.pipelines.multi_modal.gridvlp_pipeline import \
+    GridVlpEmbeddingPipeline
+
+    >>> pipeline = GridVlpEmbeddingPipeline('rgtjf1/multi-modal_gridvlp_classification_chinese-base-ecom-embedding')
+    >>> outputs = pipeline({'text': '女装快干弹力轻型短裤448575',\
+        'image':'https://yejiabo-public.oss-cn-zhangjiakou.aliyuncs.com/alinlp/clothes.png'})
+    >>> outputs["text_embedding"].shape
+    (768,)
+
+    ```
+    """
+
+    def forward(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        s4 = time.time()
+
+        box_tensor = torch.zeros(1, dtype=torch.float32)
+
+        output = self.model(
+            torch.tensor(inputs['image']).unsqueeze(0),
+            box_tensor.unsqueeze(0),
+            torch.tensor(inputs['input_ids'], dtype=torch.long).unsqueeze(0),
+            torch.tensor(inputs['input_mask'], dtype=torch.long).unsqueeze(0),
+            torch.tensor(inputs['segment_ids'], dtype=torch.long).unsqueeze(0))
+        s5 = time.time()
+
+        output = output[0].detach().numpy()
+
+        s6 = time.time()
+        logger.info(f'forward. Infer:{cost(s5, s4)}, Post: {cost(s6, s5)}')
+        # 返回结果
+        return output
+
+    def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        outputs = {
+            'img_embedding': inputs,
+            'text_embedding': inputs,
+        }
+        return outputs

tests/pipelines/test_gridvlp_classification.py

@@ -0,0 +1,71 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import unittest
+
+from modelscope.pipelines.multi_modal.gridvlp_pipeline import (
+    GridVlpClassificationPipeline, GridVlpEmbeddingPipeline)
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
+from modelscope.utils.test_utils import test_level
+
+
+class GridVlpClassificationTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.model_id = 'rgtjf1/multi-modal_gridvlp_classification_chinese-base-ecom-cate'
+
+    text = '女装快干弹力轻型短裤448575'
+    image = 'https://yejiabo-public.oss-cn-zhangjiakou.aliyuncs.com/alinlp/clothes.png'
+
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_run_with_pipeline(self):
+
+        gridvlp_classification_pipeline = GridVlpClassificationPipeline(
+            'rgtjf1/multi-modal_gridvlp_classification_chinese-base-ecom-cate')
+        input_params = {'text': self.text, 'image': self.image}
+        inputs = gridvlp_classification_pipeline.preprocess(input_params)
+        outputs = gridvlp_classification_pipeline.forward(inputs)
+        print(f'text: {self.text}\nimage: {self.image}\n'
+              f'outputs: {outputs}')
+
+        gridvlp_classification_pipeline = GridVlpClassificationPipeline(
+            'rgtjf1/multi-modal_gridvlp_classification_chinese-base-ecom-cate-large'
+        )
+        input_params = {'text': self.text, 'image': self.image}
+        inputs = gridvlp_classification_pipeline.preprocess(input_params)
+        outputs = gridvlp_classification_pipeline.forward(inputs)
+        print(f'text: {self.text}\nimage: {self.image}\n'
+              f'outputs: {outputs}')
+
+        gridvlp_classification_pipeline = GridVlpClassificationPipeline(
+            'rgtjf1/multi-modal_gridvlp_classification_chinese-base-ecom-brand'
+        )
+        input_params = {'text': self.text, 'image': self.image}
+        inputs = gridvlp_classification_pipeline.preprocess(input_params)
+        outputs = gridvlp_classification_pipeline.forward(inputs)
+        print(f'text: {self.text}\nimage: {self.image}\n'
+              f'outputs: {outputs}')
+
+        gridvlp_classification_pipeline = GridVlpClassificationPipeline(
+            'rgtjf1/multi-modal_gridvlp_classification_chinese-base-similarity'
+        )
+        input_params = {'text': self.text, 'image': self.image}
+        inputs = gridvlp_classification_pipeline.preprocess(input_params)
+        outputs = gridvlp_classification_pipeline.forward(inputs)
+        print(f'text: {self.text}\nimage: {self.image}\n'
+              f'outputs: {outputs}')
+
+        gridvlp_embedding_pipeline = GridVlpEmbeddingPipeline(
+            'rgtjf1/multi-modal_gridvlp_classification_chinese-base-ecom-embedding'
+        )
+        input_params = {'text': self.text, 'image': self.image}
+        inputs = gridvlp_embedding_pipeline.preprocess(input_params)
+        outputs = gridvlp_embedding_pipeline.forward(inputs)
+        print(f'text: {self.text}\nimage: {self.image}\n'
+              f'outputs shape: {outputs.shape}')
+
+    @unittest.skip('demo compatibility test is only enabled on a needed-basis')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
+
+if __name__ == '__main__':
+    unittest.main()