add prost text video retrival model

https://www.modelscope.cn/models/damo/multi_modal_clip_vtretrieval_prost/summary

Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/13487540

modelscope/fileio/format/jsonplus.py

@@ -4,30 +4,29 @@
 # TODO: handle environments without threads
 # (Python compiled without thread support)
 
+import numpy as np
 import simplejson as json
-from operator import attrgetter
-from sortedcontainers import SortedList
-from datetime import datetime, timedelta, date, time
-from dateutil.parser import parse as parse_datetime
-from functools import wraps, partial
-from operator import methodcaller
-from decimal import Decimal
-from fractions import Fraction
-from collections import namedtuple
 import threading
 import uuid
-import numpy as np
+from collections import namedtuple
+from datetime import date, datetime, time, timedelta
+from dateutil.parser import parse as parse_datetime
+from decimal import Decimal
+from fractions import Fraction
+from functools import partial, wraps
+from operator import attrgetter, methodcaller
+from sortedcontainers import SortedList
 
 try:
-    from moneyed import Money, Currency
+    from moneyed import Currency, Money
 except ImportError:
     # defer failing to actual (de-)serialization
     pass
 
-__all__ = ["loads", "dumps", "pretty",
-           "json_loads", "json_dumps", "json_prettydump",
-           "encoder", "decoder"]
-
+__all__ = [
+    "loads", "dumps", "pretty", "json_loads", "json_dumps", "json_prettydump",
+    "encoder", "decoder"
+]
 
 # Should we aim for the *exact* reproduction of Python types,
 # or for maximum *compatibility* when (de-)serializing?
@@ -59,12 +58,15 @@ CODING_DEFAULT = EXACT
 
 _local = threading.local()
 
+
 def prefer(coding):
     _local.coding = coding
 
+
 def prefer_exact():
     prefer(EXACT)
 
+
 def prefer_compat():
     prefer(COMPAT)
 
@@ -103,15 +105,18 @@ def kwargified(constructor):
         >>> test({'b': 3})
         4
     """
+
     @wraps(constructor)
     def kwargs_constructor(kwargs):
         return constructor(**kwargs)
+
     return kwargs_constructor
 
 
 _PredicatedEncoder = namedtuple('_PredicatedEncoder',
                                 'priority predicate encoder typename')
 
+
 def encoder(classname, predicate=None, priority=None, exact=True):
     """A decorator for registering a new encoder for object type
     defined either by a `classname`, or detected via `predicate`.
@@ -182,14 +187,18 @@ def _json_default_exact(obj):
     # first try predicate-based encoders
     for handler in _encode_handlers['exact']['predicate']:
         if handler.predicate(obj):
-            return {"__class__": handler.typename,
-                    "__value__": handler.encoder(obj)}
+            return {
+                "__class__": handler.typename,
+                "__value__": handler.encoder(obj)
+            }
 
     # then classname-based
     classname = type(obj).__name__
     if classname in _encode_handlers['exact']['classname']:
-        return {"__class__": classname,
-                "__value__": _encode_handlers['exact']['classname'][classname](obj)}
+        return {
+            "__class__": classname,
+            "__value__": _encode_handlers['exact']['classname'][classname](obj)
+        }
 
     raise TypeError(repr(obj) + " is not JSON serializable")
 
@@ -217,8 +226,10 @@ def decoder(classname):
         def mytype_decoder(value):
             return mytype(value, reconstruct=True)
     """
+
     def _decorator(f):
         _decode_handlers.setdefault(classname, f)
+
     return _decorator
 
 
@@ -235,25 +246,25 @@ def _json_object_hook(dict):
     return dict
 
 
-
 def _encoder_default_args(kw):
     """Shape default arguments for encoding functions."""
-    
+
     # manual override of the preferred coding with `exact=False`
     if kw.pop('exact', getattr(_local, 'coding', CODING_DEFAULT) == EXACT):
         # settings necessary for the "exact coding"
         kw.update({
             'default': _json_default_exact,
-            'use_decimal': False,           # don't encode `Decimal` as JSON's `Number`
-            'tuple_as_array': False,        # don't encode `tuple` as `Array`
-            'namedtuple_as_object': False   # don't call `_asdict` on `namedtuple`
+            'use_decimal': False,  # don't encode `Decimal` as JSON's `Number`
+            'tuple_as_array': False,  # don't encode `tuple` as `Array`
+            'namedtuple_as_object':
+            False  # don't call `_asdict` on `namedtuple`
         })
     else:
         # settings for the "compatibility coding"
         kw.update({
             'default': _json_default_compat,
-            'ignore_nan': True      # be compliant with the ECMA-262 specification:
-                                    # serialize nan/inf as null
+            'ignore_nan': True  # be compliant with the ECMA-262 specification:
+            # serialize nan/inf as null
         })
 
     # NOTE: if called from ``simplejson.dumps()`` with ``cls=JSONEncoder``,
@@ -276,8 +287,8 @@ def _decoder_default_args(kw):
     kw.update({'object_hook': _json_object_hook})
 
 
-
 class JSONEncoder(json.JSONEncoder):
+
     def __init__(self, **kw):
         """Constructor for simplejson.JSONEncoder, with defaults overriden
         for jsonplus.
@@ -287,6 +298,7 @@ class JSONEncoder(json.JSONEncoder):
 
 
 class JSONDecoder(json.JSONDecoder):
+
     def __init__(self, **kw):
         """Constructor for simplejson.JSONDecoder, with defaults overriden
         for jsonplus.
@@ -295,7 +307,6 @@ class JSONDecoder(json.JSONDecoder):
         super(JSONDecoder, self).__init__(**kw)
 
 
-
 def dumps(*pa, **kw):
     _encoder_default_args(kw)
     return json.dumps(*pa, **kw)
@@ -306,14 +317,13 @@ def loads(*pa, **kw):
     return json.loads(*pa, **kw)
 
 
-def pretty(x, sort_keys=True, indent=4*' ', separators=(',', ': '), **kw):
+def pretty(x, sort_keys=True, indent=4 * ' ', separators=(',', ': '), **kw):
     kw.setdefault('sort_keys', sort_keys)
     kw.setdefault('indent', indent)
     kw.setdefault('separators', separators)
     return dumps(x, **kw)
 
 
-
 json_dumps = dumps
 json_loads = loads
 json_prettydump = pretty
@@ -330,21 +340,36 @@ def generic_to_item(value):
 _encode_handlers = {
     'exact': {
         'classname': {
-            'datetime': methodcaller('isoformat'),
-            'date': methodcaller('isoformat'),
-            'time': methodcaller('isoformat'),
-            'timedelta': partial(getattrs, attrs=['days', 'seconds', 'microseconds']),
-            'tuple': list,
-            'set': list,
-            'ndarray': np_to_list,
-            'float16': generic_to_item,
-            'float32': generic_to_item,
-            'frozenset': list,
-            'complex': partial(getattrs, attrs=['real', 'imag']),
-            'Decimal': str,
-            'Fraction': partial(getattrs, attrs=['numerator', 'denominator']),
-            'UUID': partial(getattrs, attrs=['hex']),
-            'Money': partial(getattrs, attrs=['amount', 'currency'])
+            'datetime':
+            methodcaller('isoformat'),
+            'date':
+            methodcaller('isoformat'),
+            'time':
+            methodcaller('isoformat'),
+            'timedelta':
+            partial(getattrs, attrs=['days', 'seconds', 'microseconds']),
+            'tuple':
+            list,
+            'set':
+            list,
+            'ndarray':
+            np_to_list,
+            'float16':
+            generic_to_item,
+            'float32':
+            generic_to_item,
+            'frozenset':
+            list,
+            'complex':
+            partial(getattrs, attrs=['real', 'imag']),
+            'Decimal':
+            str,
+            'Fraction':
+            partial(getattrs, attrs=['numerator', 'denominator']),
+            'UUID':
+            partial(getattrs, attrs=['hex']),
+            'Money':
+            partial(getattrs, attrs=['amount', 'currency'])
         },
         'predicate': SortedList(key=attrgetter('priority'))
     },
@@ -368,7 +393,6 @@ _encode_handlers = {
     }
 }
 
-
 # all decode handlers are for EXACT decoding BY CLASSNAME
 _decode_handlers = {
     'datetime': parse_datetime,
@@ -388,11 +412,14 @@ _decode_handlers = {
 }
 
 
-@encoder('namedtuple', lambda obj: isinstance(obj, tuple) and hasattr(obj, '_fields'))
+@encoder('namedtuple',
+         lambda obj: isinstance(obj, tuple) and hasattr(obj, '_fields'))
 def _dump_namedtuple(obj):
-    return {"name": type(obj).__name__,
-            "fields": list(obj._fields),
-            "values": list(obj)}
+    return {
+        "name": type(obj).__name__,
+        "fields": list(obj._fields),
+        "values": list(obj)
+    }
 
 
 @decoder('namedtuple')
@@ -404,7 +431,8 @@ def _load_namedtuple(val):
 @encoder('timedelta', exact=False)
 def _timedelta_total_seconds(td):
     # timedelta.total_seconds() is only available since python 2.7
-    return (td.microseconds + (td.seconds + td.days * 24 * 3600.0) * 10**6) / 10**6
+    return (td.microseconds +
+            (td.seconds + td.days * 24 * 3600.0) * 10**6) / 10**6
 
 
 @encoder('Currency')
@@ -412,7 +440,7 @@ def _dump_currency(obj):
     """Serialize standard (ISO-defined) currencies to currency code only,
     and non-standard (user-added) currencies in full.
     """
-    from moneyed import get_currency, CurrencyDoesNotExist
+    from moneyed import CurrencyDoesNotExist, get_currency
     try:
         get_currency(obj.code)
         return obj.code

modelscope/metainfo.py

@@ -213,6 +213,7 @@ class Models(object):
     video_synthesis = 'latent-text-to-video-synthesis'
     team = 'team-multi-modal-similarity'
     video_clip = 'video-clip-multi-modal-embedding'
+    prost = 'prost-clip-text-video-retrieval'
     mgeo = 'mgeo'
     vldoc = 'vldoc'
     hitea = 'hitea'
@@ -528,6 +529,7 @@ class Pipelines(object):
     multi_modal_similarity = 'multi-modal-similarity'
     text_to_image_synthesis = 'text-to-image-synthesis'
     video_multi_modal_embedding = 'video-multi-modal-embedding'
+    prost_text_video_retrieval = 'prost-text-video-retrieval'
     image_text_retrieval = 'image-text-retrieval'
     ofa_ocr_recognition = 'ofa-ocr-recognition'
     ofa_asr = 'ofa-asr'
@@ -720,6 +722,8 @@ DEFAULT_MODEL_FOR_PIPELINE = {
     Tasks.video_multi_modal_embedding:
     (Pipelines.video_multi_modal_embedding,
      'damo/multi_modal_clip_vtretrival_msrvtt_53'),
+    Tasks.text_video_retrieval: (Pipelines.prost_text_video_retrieval,
+                                 'damo/multi_modal_clip_vtretrieval_prost'),
     Tasks.image_color_enhancement:
     (Pipelines.image_color_enhance,
      'damo/cv_csrnet_image-color-enhance-models'),

modelscope/models/multi_modal/__init__.py

@@ -6,22 +6,23 @@ from modelscope.utils.import_utils import LazyImportModule
 if TYPE_CHECKING:
 
     from .clip import CLIPForMultiModalEmbedding
-    from .gemm import GEMMForMultiModalEmbedding
-    from .rleg import RLEGForMultiModalEmbedding
-    from .team import TEAMForMultiModalSimilarity
+    from .clip_interrogator import CLIP_Interrogator
     from .diffusion import DiffusionForTextToImageSynthesis
+    from .efficient_diffusion_tuning import EfficientStableDiffusion
+    from .gemm import GEMMForMultiModalEmbedding
     from .mmr import VideoCLIPForMultiModalEmbedding
-    from .mplug_for_all_tasks import MPlugForAllTasks, HiTeAForAllTasks
+    from .mplug_for_all_tasks import HiTeAForAllTasks, MPlugForAllTasks
+    from .mplug_owl import MplugOwlForConditionalGeneration
+    from .multi_stage_diffusion import \
+        MultiStageDiffusionForTextToImageSynthesis
     from .ofa_for_all_tasks import OfaForAllTasks
     from .ofa_for_text_to_image_synthesis_model import \
         OfaForTextToImageSynthesis
-    from .multi_stage_diffusion import \
-        MultiStageDiffusionForTextToImageSynthesis
-    from .vldoc import VLDocForDocVLEmbedding
+    from .prost import ProSTForTVRetrieval
+    from .rleg import RLEGForMultiModalEmbedding
+    from .team import TEAMForMultiModalSimilarity
     from .video_synthesis import TextToVideoSynthesis
-    from .efficient_diffusion_tuning import EfficientStableDiffusion
-    from .mplug_owl import MplugOwlForConditionalGeneration
-    from .clip_interrogator import CLIP_Interrogator
+    from .vldoc import VLDocForDocVLEmbedding
 
 else:
     _import_structure = {
@@ -31,6 +32,7 @@ else:
         'rleg': ['RLEGForMultiModalEmbedding'],
         'team': ['TEAMForMultiModalSimilarity'],
         'mmr': ['VideoCLIPForMultiModalEmbedding'],
+        'prost': ['ProSTForTVRetrieval'],
         'mplug_for_all_tasks': ['MPlugForAllTasks', 'HiTeAForAllTasks'],
         'ofa_for_all_tasks': ['OfaForAllTasks'],
         'ofa_for_text_to_image_synthesis_model':

modelscope/models/multi_modal/prost/__init__.py

@@ -0,0 +1,3 @@
+# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved.
+
+from .models import ProSTForTVRetrieval

modelscope/models/multi_modal/prost/dataloaders/rawvideo_util.py

@@ -0,0 +1,117 @@
+# The implementation is adopted from Huaishao Luo,
+# made pubicly available under the MIT License at https://github.com/ArrowLuo/CLIP4Clip
+
+import cv2
+import numpy as np
+import torch as th
+from PIL import Image
+from torchvision.transforms import (CenterCrop, Compose, InterpolationMode,
+                                    Normalize, Resize, ToTensor)
+
+from modelscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class RawVideoExtractorCV2():
+
+    def __init__(
+        self,
+        centercrop=False,
+        size=224,
+        frame_rate=-1,
+    ):
+        self.centercrop = centercrop
+        self.size = size
+        self.framerate = frame_rate
+        self.transform = self._transform(self.size)
+
+    def _transform(self, n_px):
+        return Compose([
+            Resize(n_px, interpolation=InterpolationMode.BICUBIC),
+            CenterCrop(n_px),
+            lambda image: image.convert('RGB'),
+            ToTensor(),
+            Normalize((0.48145466, 0.4578275, 0.40821073),
+                      (0.26862954, 0.26130258, 0.27577711)),
+        ])
+
+    def video_to_tensor(self,
+                        video_file,
+                        preprocess,
+                        sample_fp=0,
+                        start_time=None,
+                        end_time=None):
+        if start_time is not None or end_time is not None:
+            assert isinstance(start_time, int) and isinstance(end_time, int) \
+                   and start_time > -1 and end_time > start_time
+        assert sample_fp > -1
+
+        # Samples a frame sample_fp X frames.
+        cap = cv2.VideoCapture(video_file)
+        frameCount = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+        fps = int(cap.get(cv2.CAP_PROP_FPS))
+
+        if fps == 0:
+            logger.info(f'{video_file} with fps 0!!!')
+        total_duration = (frameCount + fps - 1) // fps
+        start_sec, end_sec = 0, total_duration
+
+        if start_time is not None:
+            start_sec, end_sec = start_time, end_time if end_time <= total_duration else total_duration
+            cap.set(cv2.CAP_PROP_POS_FRAMES, int(start_time * fps))
+
+        interval = 1
+        if sample_fp > 0:
+            interval = fps // sample_fp
+        else:
+            sample_fp = fps
+        if interval == 0:
+            interval = 1
+
+        inds = [ind for ind in np.arange(0, fps, interval)]
+        assert len(inds) >= sample_fp
+        inds = inds[:sample_fp]
+
+        ret = True
+        images = []
+
+        for sec in np.arange(start_sec, end_sec + 1):
+            if not ret:
+                break
+            sec_base = int(sec * fps)
+            for ind in inds:
+                cap.set(cv2.CAP_PROP_POS_FRAMES, sec_base + ind)
+                ret, frame = cap.read()
+                if not ret:
+                    break
+                frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+                images.append(
+                    preprocess(Image.fromarray(frame_rgb).convert('RGB')))
+
+        cap.release()
+
+        if len(images) > 0:
+            video_data = th.tensor(np.stack(images))
+        else:
+            video_data = th.zeros(1)
+        return {'video': video_data}
+
+    def get_video_data(self, video_path, start_time=None, end_time=None):
+        image_input = self.video_to_tensor(
+            video_path,
+            self.transform,
+            sample_fp=self.framerate,
+            start_time=start_time,
+            end_time=end_time)
+        return image_input
+
+    def process_raw_data(self, raw_video_data):
+        tensor_size = raw_video_data.size()
+        tensor = raw_video_data.view(-1, 1, tensor_size[-3], tensor_size[-2],
+                                     tensor_size[-1])
+        return tensor
+
+
+# An ordinary video frame extractor based CV2
+RawVideoExtractor = RawVideoExtractorCV2

modelscope/models/multi_modal/prost/models/__init__.py

@@ -0,0 +1,3 @@
+# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved.
+
+from .prost_model import ProSTForTVRetrieval

modelscope/models/multi_modal/prost/models/modeling.py

@@ -0,0 +1,704 @@
+# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved.
+
+import os
+import platform
+from collections import OrderedDict
+from types import SimpleNamespace
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
+
+from modelscope.models.multi_modal.prost.models.module_clip import (
+    _PT_NAME, CLIP, QuickGELU, convert_weights)
+from modelscope.models.multi_modal.prost.models.module_cross import (
+    CrossConfig, CrossModel)
+from modelscope.models.multi_modal.prost.models.module_cross import \
+    Transformer as TransformerClip
+from modelscope.models.multi_modal.prost.models.until_module import (
+    AllGather, CrossEn, Event_decoder, Frame_decoder, LayerNorm,
+    PreTrainedModel, make_patch_shift)
+from modelscope.utils.logger import get_logger
+
+allgather = AllGather.apply
+
+logger = get_logger()
+__all__ = ['CLIP4Clip']
+
+
+class MyObject:
+
+    def __init__(self, **kwargs):
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+
+
+class CLIP4ClipPreTrainedModel(PreTrainedModel, nn.Module):
+    """ An abstract class to handle weights initialization and
+        a simple interface for dowloading and loading pretrained models.
+    """
+
+    def __init__(self, cross_config, *inputs, **kwargs):
+        super(CLIP4ClipPreTrainedModel, self).__init__(cross_config)
+        self.cross_config = cross_config
+        self.clip = None
+        self.cross = None
+
+    @classmethod
+    def from_pretrained(cls,
+                        cross_config,
+                        state_dict=None,
+                        cache_dir=None,
+                        type_vocab_size=2,
+                        *inputs,
+                        **kwargs):
+
+        task_config = None
+        if 'task_config' in kwargs.keys():
+            task_config = kwargs['task_config']
+            if not hasattr(task_config, 'local_rank'):
+                task_config['local_rank'] = 0
+            elif task_config['local_rank'] == -1:
+                task_config['local_rank'] = 0
+
+        if state_dict is None:
+            state_dict = {}
+        # pretrained_clip_name = task_config['pretrained_clip_name']
+        clip_state_dict = CLIP.get_config(model_dir=task_config['model_dir'])
+        for key, val in clip_state_dict.items():
+            new_key = 'clip.' + key
+            if new_key not in state_dict:
+                state_dict[new_key] = val.clone()
+
+        # cross_config, _ = CrossConfig.get_config(
+        #     cross_model_name,
+        #     cache_dir,
+        #     type_vocab_size,
+        #     state_dict=None,
+        #     task_config=task_config)
+        cross_config = CrossConfig.from_dict(cross_config)
+        cross_config.type_vocab_size = type_vocab_size
+        task_config = MyObject(**kwargs['task_config'])
+        model = cls(cross_config, clip_state_dict, *inputs, task_config)
+
+        # ===> Initialization trick [HARD CODE]
+        if model.linear_patch == '3d':
+            contain_conv2 = False
+            for key in state_dict.keys():
+                if key.find('visual.conv2.weight') > -1:
+                    contain_conv2 = True
+                    break
+            if contain_conv2 is False and hasattr(model.clip.visual, 'conv2'):
+                cp_weight = state_dict['clip.visual.conv1.weight'].clone()
+                kernel_size = model.clip.visual.conv2.weight.size(2)
+                conv2_size = model.clip.visual.conv2.weight.size()
+                conv2_size = list(conv2_size)
+
+                left_conv2_size = conv2_size.copy()
+                right_conv2_size = conv2_size.copy()
+                left_conv2_size[2] = (kernel_size - 1) // 2
+                right_conv2_size[2] = kernel_size - 1 - left_conv2_size[2]
+
+                left_zeros, right_zeros = None, None
+                if left_conv2_size[2] > 0:
+                    left_zeros = torch.zeros(
+                        *tuple(left_conv2_size),
+                        dtype=cp_weight.dtype,
+                        device=cp_weight.device)
+                if right_conv2_size[2] > 0:
+                    right_zeros = torch.zeros(
+                        *tuple(right_conv2_size),
+                        dtype=cp_weight.dtype,
+                        device=cp_weight.device)
+
+                cat_list = []
+                if left_zeros is not None:
+                    cat_list.append(left_zeros)
+                cat_list.append(cp_weight.unsqueeze(2))
+                if right_zeros is not None:
+                    cat_list.append(right_zeros)
+                cp_weight = torch.cat(cat_list, dim=2)
+
+                state_dict['clip.visual.conv2.weight'] = cp_weight
+
+        # if model.sim_header == 'tightTransf':
+        #     contain_cross = False
+        #     for key in state_dict.keys():
+        #         if key.find('cross.transformer') > -1:
+        #             contain_cross = True
+        #             break
+        #     if contain_cross is False:
+        #         for key, val in clip_state_dict.items():
+        #             if key == 'positional_embedding':
+        #                 state_dict[
+        #                     'cross.embeddings.position_embeddings.weight'] = val.clone(
+        #                     )
+        #                 continue
+        #             if key.find('transformer.resblocks') == 0:
+        #                 num_layer = int(key.split('.')[2])
+
+        #                 # cut from beginning
+        #                 if num_layer < task_config.cross_num_hidden_layers:
+        #                     state_dict['cross.' + key] = val.clone()
+        #                     continue
+
+        if model.sim_header == 'seqLSTM' or model.sim_header == 'seqTransf':
+            # This step is to detect whether in train mode or test mode
+            contain_frame_position = False
+            for key in state_dict.keys():
+                if key.find('frame_position_embeddings') > -1:
+                    contain_frame_position = True
+                    break
+
+            # train mode
+            if contain_frame_position is False:
+                for key, val in clip_state_dict.items():
+                    if key == 'positional_embedding':
+                        state_dict[
+                            'frame_position_embeddings.weight'] = val.clone()
+                        # state_dict["text_prompt_encoder.pos_embedding"] = val[0:3].clone()
+                        continue
+                    if model.sim_header == 'seqTransf' and key.find(
+                            'transformer.resblocks') == 0:
+                        num_layer = int(key.split('.')[2])
+                        # cut from beginning
+                        if num_layer < task_config.cross_num_hidden_layers:
+                            state_dict[key.replace(
+                                'transformer.',
+                                'transformerClip.')] = val.clone()
+                            continue
+
+            else:
+                for key, val in state_dict.items():
+                    # test mode
+                    if key.find('clip.visual.transformer.resblocks') == 0:
+                        num_layer = int(key.split('.')[4])
+                        # shift layers 10-11
+                        if num_layer >= 10 and num_layer < 12:
+                            state_dict[key.replace('attn.net.',
+                                                   'attn.')] = val.clone()
+        # <=== End of initialization trick
+
+        if state_dict is not None:
+            model = cls.init_preweight(
+                model, state_dict, task_config=task_config)
+        make_patch_shift(model, video_frame=task_config.max_frames, n_div=14)
+        return model
+
+
+def show_log(task_config, info):
+    if task_config is None or task_config.local_rank == 0:
+        logger.warning(info)
+
+
+def update_attr(target_name,
+                target_config,
+                target_attr_name,
+                source_config,
+                source_attr_name,
+                default_value=None):
+    if hasattr(source_config, source_attr_name):
+        if default_value is None or getattr(source_config,
+                                            source_attr_name) != default_value:
+            setattr(target_config, target_attr_name,
+                    getattr(source_config, source_attr_name))
+            # show_log(
+            #     source_config, "Set {}.{}: {}.".format(
+            #         target_name, target_attr_name,
+            #         getattr(target_config, target_attr_name)))
+    return target_config
+
+
+def check_attr(target_name, task_config):
+    return hasattr(task_config,
+                   target_name) and task_config.__dict__[target_name]
+
+
+class CLIP4Clip(CLIP4ClipPreTrainedModel):
+
+    def __init__(self, cross_config, clip_state_dict, task_config):
+        super(CLIP4Clip, self).__init__(cross_config)
+        self.task_config = task_config
+        self.ignore_video_index = -1
+
+        assert self.task_config.max_words + self.task_config.max_frames <= cross_config.max_position_embeddings
+
+        self._stage_one = True
+        self._stage_two = False
+
+        # show_log(task_config, "Stage-One:{}, Stage-Two:{}".format(self._stage_one, self._stage_two))
+
+        self.loose_type = False
+        if self._stage_one and check_attr('loose_type', self.task_config):
+            self.loose_type = True
+            # show_log(task_config, "Test retrieval by loose type.")
+
+        # CLIP Encoders: From OpenAI: CLIP [https://github.com/openai/CLIP] ===>
+        vit = 'visual.proj' in clip_state_dict
+        assert vit
+        if vit:
+            vision_width = clip_state_dict['visual.conv1.weight'].shape[0]
+            vision_layers = len([
+                k for k in clip_state_dict.keys() if k.startswith('visual.')
+                and k.endswith('.attn.in_proj_weight')
+            ])
+            vision_patch_size = clip_state_dict['visual.conv1.weight'].shape[
+                -1]
+            grid_size = round(
+                (clip_state_dict['visual.positional_embedding'].shape[0]
+                 - 1)**0.5)
+            image_resolution = vision_patch_size * grid_size
+        else:
+            counts: list = [
+                len(
+                    set(
+                        k.split('.')[2] for k in clip_state_dict
+                        if k.startswith(f'visual.layer{b}')))
+                for b in [1, 2, 3, 4]
+            ]
+            vision_layers = tuple(counts)
+            vision_width = clip_state_dict[
+                'visual.layer1.0.conv1.weight'].shape[0]
+            output_width = round(
+                (clip_state_dict['visual.attnpool.positional_embedding'].
+                 shape[0] - 1)**0.5)
+            vision_patch_size = None
+            assert output_width**2 + 1 == clip_state_dict[
+                'visual.attnpool.positional_embedding'].shape[0]
+            image_resolution = output_width * 32
+
+        embed_dim = clip_state_dict['text_projection'].shape[1]
+        context_length = clip_state_dict['positional_embedding'].shape[0]
+        vocab_size = clip_state_dict['token_embedding.weight'].shape[0]
+        transformer_width = clip_state_dict['ln_final.weight'].shape[0]
+        transformer_heads = transformer_width // 64
+        transformer_layers = len(
+            set(
+                k.split('.')[2] for k in clip_state_dict
+                if k.startswith('transformer.resblocks')))
+
+        # show_log(task_config, "\t embed_dim: {}".format(embed_dim))
+        # show_log(task_config, "\t image_resolution: {}".format(image_resolution))
+        # show_log(task_config, "\t vision_layers: {}".format(vision_layers))
+        # show_log(task_config, "\t vision_width: {}".format(vision_width))
+        # show_log(task_config, "\t vision_patch_size: {}".format(vision_patch_size))
+        # show_log(task_config, "\t context_length: {}".format(context_length))
+        # show_log(task_config, "\t vocab_size: {}".format(vocab_size))
+        # show_log(task_config, "\t transformer_width: {}".format(transformer_width))
+        # show_log(task_config, "\t transformer_heads: {}".format(transformer_heads))
+        # show_log(task_config, "\t transformer_layers: {}".format(transformer_layers))
+
+        self.linear_patch = '2d'
+        if hasattr(task_config, 'linear_patch'):
+            self.linear_patch = task_config.linear_patch
+            # show_log(task_config, "\t\t linear_patch: {}".format(self.linear_patch))
+
+        # use .float() to avoid overflow/underflow from fp16 weight. https://github.com/openai/CLIP/issues/40
+        cut_top_layer = 0
+
+        self.clip = CLIP(
+            embed_dim,
+            image_resolution,
+            vision_layers - cut_top_layer,
+            vision_width,
+            vision_patch_size,
+            context_length,
+            vocab_size,
+            transformer_width,
+            transformer_heads,
+            transformer_layers - cut_top_layer,
+            linear_patch=self.linear_patch).float()
+
+        for key in ['input_resolution', 'context_length', 'vocab_size']:
+            if key in clip_state_dict:
+                del clip_state_dict[key]
+
+        convert_weights(self.clip)
+        # <=== End of CLIP Encoders
+
+        self.sim_header = 'seqTransf'
+        if hasattr(task_config, 'sim_header'):
+            self.sim_header = task_config.sim_header
+            # show_log(task_config, "\t sim_header: {}".format(self.sim_header))
+        if self.sim_header == 'tightTransf':
+            assert self.loose_type is False
+
+        cross_config.max_position_embeddings = context_length
+        if self.loose_type is False:
+            # Cross Encoder ===>
+            cross_config = update_attr('cross_config', cross_config,
+                                       'num_hidden_layers', self.task_config,
+                                       'cross_num_hidden_layers')
+            self.cross = CrossModel(cross_config)
+            # <=== End of Cross Encoder
+            self.similarity_dense = nn.Linear(cross_config.hidden_size, 1)
+
+        if self.sim_header == 'seqLSTM' or self.sim_header == 'seqTransf':
+            self.frame_position_embeddings = nn.Embedding(
+                cross_config.max_position_embeddings, cross_config.hidden_size)
+            # self.frame_position_embeddings = nn.Embedding(600, cross_config.hidden_size)
+        if self.sim_header == 'seqTransf':
+            self.transformerClip = TransformerClip(
+                width=transformer_width,
+                layers=self.task_config.cross_num_hidden_layers,
+                heads=transformer_heads,
+            )
+        if self.sim_header == 'seqLSTM':
+            self.lstm_visual = nn.LSTM(
+                input_size=cross_config.hidden_size,
+                hidden_size=cross_config.hidden_size,
+                batch_first=True,
+                bidirectional=False,
+                num_layers=1)
+
+        self.loss_fct = CrossEn()
+        self.apply(self.init_weights)
+
+        self.set_dim = 512
+        self.patch_num = self.task_config.max_patch
+        if hasattr(self.task_config, 'max_word_pro'):
+            self.word_pro_num = self.task_config.max_word_pro
+        else:
+            self.word_pro_num = self.task_config.max_phrase
+
+        self.frame_num = self.task_config.max_frames
+        if hasattr(self.task_config, 'max_vfea'):
+            self.event_num = self.task_config.max_vfea
+        else:
+            self.event_num = self.task_config.max_event
+
+        self.patch_prototype_weight = nn.Sequential(
+            nn.Linear(self.set_dim, self.set_dim), nn.ReLU(inplace=True),
+            nn.Linear(self.set_dim, self.patch_num - 1), nn.ReLU(inplace=True))
+
+        self.word_prototype_weight = nn.Sequential(
+            nn.Linear(self.set_dim, self.set_dim), nn.ReLU(inplace=True),
+            nn.Linear(self.set_dim, self.word_pro_num), nn.ReLU(inplace=True))
+
+        self.frame_decoder = Frame_decoder(
+            num_attris=self.frame_num,
+            layers=2,
+            heads=1,
+            dim_ftr=512,
+            pos_emb=False,
+            length=1,
+            dim_feedforward=512,
+            without_init=False)
+        self.event_decoder = Event_decoder(
+            num_attris=self.event_num,
+            layers=2,
+            heads=1,
+            dim_ftr=512,
+            pos_emb=False,
+            length=1,
+            dim_feedforward=512,
+            without_init=False)
+        # -------------------------------------------------------------------------------------------------------
+
+    def forward(self,
+                input_ids,
+                token_type_ids,
+                attention_mask,
+                video,
+                video_mask=None):
+        input_ids = input_ids.view(-1, input_ids.shape[-1])
+        token_type_ids = token_type_ids.view(-1, token_type_ids.shape[-1])
+        attention_mask = attention_mask.view(-1, attention_mask.shape[-1])
+        video_mask = video_mask.view(-1, video_mask.shape[-1])
+
+        # T x 3 x H x W
+        video = torch.as_tensor(video).float()
+        bs, ts, channel, h, w = video.shape
+        video = video.view(bs * ts, channel, h, w)
+        video_frame = bs * ts
+        phr_feat, sen_feat, obj_feat, eve_feat = self.get_sequence_visual_output(
+            input_ids,
+            token_type_ids,
+            attention_mask,
+            video,
+            video_mask,
+            shaped=True,
+            video_frame=video_frame)
+
+        if self.training:
+            sim_matrix1, sim_matrix2, sim_matrix3, sim_matrix4 = self.get_max_similarity_logits(
+                phr_feat,
+                sen_feat,
+                obj_feat,
+                eve_feat,
+                attention_mask,
+                video_mask,
+                shaped=True)
+            sim_loss = (self.loss_fct(sim_matrix1) + self.loss_fct(sim_matrix2)
+                        + self.loss_fct(sim_matrix3)
+                        + self.loss_fct(sim_matrix4)) / 4.0
+
+            loss = sim_loss
+
+            return loss
+        else:
+            return None
+
+    def get_max_similarity_logits(self,
+                                  word_feat,
+                                  text_feat,
+                                  patch_feat,
+                                  video_feat,
+                                  text_mask,
+                                  video_mask,
+                                  shaped=False):
+        if shaped is False:
+            text_mask = text_mask.view(-1, text_mask.shape[-1])
+            video_mask = video_mask.view(-1, video_mask.shape[-1])
+
+        if self.training and torch.cuda.is_available():  # batch merge here
+            text_feat = allgather(text_feat, self.task_config)
+            video_feat = allgather(video_feat, self.task_config)
+            word_feat = allgather(word_feat, self.task_config)
+            patch_feat = allgather(patch_feat, self.task_config)
+
+            video_mask = allgather(video_mask, self.task_config)
+            torch.distributed.barrier()  # force sync
+
+        # ESPM
+        text_feat = F.normalize(text_feat, p=2, dim=1)
+        video_feat = F.normalize(video_feat, p=2, dim=2)
+        retrieve_logits = torch.einsum('ad,bkd->abk', [text_feat, video_feat])
+        retrieve_logits = retrieve_logits.max(2)[0]
+
+        # OPPM
+        word_feat = F.normalize(word_feat, p=2, dim=2)
+        patch_feat = F.normalize(patch_feat, p=2, dim=3)
+        retrieve_logits_2 = torch.einsum('aid, bfjd->abfij',
+                                         [word_feat, patch_feat])
+
+        retrieve_logits_2 = retrieve_logits_2.max(3)[0]
+        retrieve_logits_2 = retrieve_logits_2.max(2)[0]
+        retrieve_logits_2 = retrieve_logits_2.sum(2) / self.patch_num
+
+        if self.training:
+            logit_scale = self.clip.logit_scale.exp()
+            retrieve_logits = logit_scale * retrieve_logits
+            retrieve_logits_2 = logit_scale * retrieve_logits_2
+        return retrieve_logits, retrieve_logits.t(
+        ), retrieve_logits_2, retrieve_logits_2.t()
+
+    def get_sequence_output(self,
+                            input_ids,
+                            token_type_ids,
+                            attention_mask,
+                            shaped=False):
+        if shaped is False:
+            input_ids = input_ids.view(-1, input_ids.shape[-1])
+            token_type_ids = token_type_ids.view(-1, token_type_ids.shape[-1])
+            attention_mask = attention_mask.view(-1, attention_mask.shape[-1])
+        bs_pair = input_ids.size(0)
+        sequence_hidden = self.clip.encode_text(
+            input_ids, return_hidden=True)[1].float()
+        text_feat = sequence_hidden.view(bs_pair, -1, sequence_hidden.size(-1))
+
+        word_weights = self.word_prototype_weight(text_feat)
+        text_word_proto = torch.einsum('bmd,bmn->bnd', text_feat, word_weights)
+
+        cls_text_feat = text_feat.contiguous()
+        cls_text_feat = cls_text_feat[torch.arange(cls_text_feat.shape[0]),
+                                      torch.sum(attention_mask, dim=-1) - 1, :]
+
+        return text_word_proto, cls_text_feat
+
+    def get_visual_output(self,
+                          video,
+                          video_mask,
+                          shaped=False,
+                          video_frame=-1):
+        if shaped is False:
+            video_mask = video_mask.view(-1, video_mask.shape[-1])
+            video = torch.as_tensor(video).float()
+            bs, ts, channel, h, w = video.shape
+            video = video.view(bs * ts, channel, h, w)
+            # video_frame = bs * ts
+
+        bs_pair = video_mask.size(0)
+
+        cls_video_feat, video_patch_feat = self.clip.encode_image_tokens(
+            video, return_hidden=True)
+        cls_video_feat = cls_video_feat.float()
+        video_patch_feat = video_patch_feat.float()
+        # frame_num = video_patch_feat.shape[0]
+        patch_dim = video_patch_feat.shape[2]
+
+        patch_weights = self.patch_prototype_weight(video_patch_feat)
+        # cls_video_feat
+        video_patch_proto = torch.einsum('bmd,bmn->bnd', video_patch_feat,
+                                         patch_weights)
+        video_patch_proto = torch.cat(
+            (cls_video_feat.unsqueeze(1), video_patch_proto), 1)
+        video_patch_proto = video_patch_proto.reshape(
+            bs_pair, self.task_config.max_frames, self.patch_num, patch_dim)
+
+        video_frame_proto = video_patch_proto.reshape(
+            bs_pair, self.patch_num * self.task_config.max_frames, patch_dim)
+        video_frame_proto = self.frame_decoder(video_frame_proto)
+
+        video_frame_proto = 0.5 * video_frame_proto + 0.5 * cls_video_feat.reshape(
+            bs_pair, self.task_config.max_frames, patch_dim)
+        video_frame_proto = self.event_decoder(video_frame_proto)
+        video_frame_proto = 0.5 * video_frame_proto + 0.5 * cls_video_feat.reshape(
+            bs_pair, self.task_config.max_frames, patch_dim).mean(1).unsqueeze(
+                1).repeat(1, video_frame_proto.shape[1], 1)
+        return video_patch_proto, video_frame_proto
+
+    def get_sequence_visual_output(self,
+                                   input_ids,
+                                   token_type_ids,
+                                   attention_mask,
+                                   video,
+                                   video_mask,
+                                   shaped=False,
+                                   video_frame=-1):
+        if shaped is False:
+            input_ids = input_ids.view(-1, input_ids.shape[-1])
+            token_type_ids = token_type_ids.view(-1, token_type_ids.shape[-1])
+            attention_mask = attention_mask.view(-1, attention_mask.shape[-1])
+            video_mask = video_mask.view(-1, video_mask.shape[-1])
+
+            video = torch.as_tensor(video).float()
+
+            # import pdb;pdb.set_trace()
+            # b, pair,
+            bs, ts, channel, h, w = video.shape
+            video = video.view(bs * ts, channel, h, w)
+            video_frame = bs * ts
+
+        word_feat, text_feat = self.get_sequence_output(
+            input_ids, token_type_ids, attention_mask, shaped=True)
+
+        patch_feat, frame_feat = self.get_visual_output(
+            video, video_mask, shaped=True, video_frame=video_frame)
+
+        return word_feat, text_feat, patch_feat, frame_feat
+
+    def _get_cross_output(self, sequence_output, visual_output, attention_mask,
+                          video_mask):
+
+        concat_features = torch.cat((sequence_output, visual_output),
+                                    dim=1)  # concatnate tokens and frames
+        concat_mask = torch.cat((attention_mask, video_mask), dim=1)
+        text_type_ = torch.zeros_like(attention_mask)
+        video_type_ = torch.ones_like(video_mask)
+        concat_type = torch.cat((text_type_, video_type_), dim=1)
+
+        cross_layers, pooled_output = self.cross(
+            concat_features,
+            concat_type,
+            concat_mask,
+            output_all_encoded_layers=True)
+        cross_output = cross_layers[-1]
+
+        return cross_output, pooled_output, concat_mask
+
+    def _mean_pooling_for_similarity_sequence(self, sequence_output,
+                                              attention_mask):
+        attention_mask_un = attention_mask.to(dtype=torch.float).unsqueeze(-1)
+        attention_mask_un[:, 0, :] = 0.
+        sequence_output = sequence_output * attention_mask_un
+        text_out = torch.sum(
+            sequence_output, dim=1) / torch.sum(
+                attention_mask_un, dim=1, dtype=torch.float)
+        return text_out
+
+    def _mean_pooling_for_similarity_visual(
+        self,
+        visual_output,
+        video_mask,
+    ):
+        video_mask_un = video_mask.to(dtype=torch.float).unsqueeze(-1)
+        visual_output = visual_output * video_mask_un
+        video_mask_un_sum = torch.sum(video_mask_un, dim=1, dtype=torch.float)
+        video_mask_un_sum[video_mask_un_sum == 0.] = 1.
+        video_out = torch.sum(visual_output, dim=1) / video_mask_un_sum
+        return video_out
+
+    def _mean_pooling_for_similarity(
+        self,
+        sequence_output,
+        visual_output,
+        attention_mask,
+        video_mask,
+    ):
+        text_out = self._mean_pooling_for_similarity_sequence(
+            sequence_output, attention_mask)
+        video_out = self._mean_pooling_for_similarity_visual(
+            visual_output, video_mask)
+
+        return text_out, video_out
+
+    def get_global_similarity(self, sequence_output, visual_output,
+                              attention_mask, video_mask):
+        visual_output = visual_output / visual_output.norm(
+            dim=-1, keepdim=True)
+        visual_output = self._mean_pooling_for_similarity_visual(
+            visual_output, video_mask)
+        visual_output = visual_output / visual_output.norm(
+            dim=-1, keepdim=True)
+
+        sequence_output = sequence_output.squeeze(1)
+        sequence_output = sequence_output / sequence_output.norm(
+            dim=-1, keepdim=True)
+
+        logit_scale = self.clip.logit_scale.exp()
+        # retrieve_logits = logit_scale * torch.matmul(sequence_output, visual_output.t())
+        sim_matrix_global = logit_scale * torch.matmul(sequence_output,
+                                                       visual_output.t())
+        return sim_matrix_global
+
+    def _cross_similarity(self, sequence_output, visual_output, attention_mask,
+                          video_mask):
+        sequence_output, visual_output = sequence_output.contiguous(
+        ), visual_output.contiguous()
+
+        b_text, s_text, h_text = sequence_output.size()
+        b_visual, s_visual, h_visual = visual_output.size()
+
+        retrieve_logits_list = []
+
+        step_size = b_text  # set smaller to reduce memory cost
+        split_size = [step_size] * (b_text // step_size)
+        release_size = b_text - sum(split_size)
+        if release_size > 0:
+            split_size += [release_size]
+
+        # due to clip text branch retrun the last hidden
+        attention_mask = torch.ones(sequence_output.size(0), 1)\
+            .to(device=attention_mask.device, dtype=attention_mask.dtype)
+
+        sequence_output_splits = torch.split(
+            sequence_output, split_size, dim=0)
+        attention_mask_splits = torch.split(attention_mask, split_size, dim=0)
+        for i in range(len(split_size)):
+            sequence_output_row = sequence_output_splits[i]
+            attention_mask_row = attention_mask_splits[i]
+            sequence_output_l = sequence_output_row.unsqueeze(1).repeat(
+                1, b_visual, 1, 1)
+            sequence_output_l = sequence_output_l.view(-1, s_text, h_text)
+            attention_mask_l = attention_mask_row.unsqueeze(1).repeat(
+                1, b_visual, 1)
+            attention_mask_l = attention_mask_l.view(-1, s_text)
+
+            step_truth = sequence_output_row.size(0)
+            visual_output_r = visual_output.unsqueeze(0).repeat(
+                step_truth, 1, 1, 1)
+            visual_output_r = visual_output_r.view(-1, s_visual, h_visual)
+            video_mask_r = video_mask.unsqueeze(0).repeat(step_truth, 1, 1)
+            video_mask_r = video_mask_r.view(-1, s_visual)
+
+            cross_output, pooled_output, concat_mask = \
+                self._get_cross_output(sequence_output_l, visual_output_r, attention_mask_l, video_mask_r)
+            retrieve_logits_row = self.similarity_dense(pooled_output).squeeze(
+                -1).view(step_truth, b_visual)
+
+            retrieve_logits_list.append(retrieve_logits_row)
+
+        retrieve_logits = torch.cat(retrieve_logits_list, dim=0)
+        return retrieve_logits

modelscope/models/multi_modal/prost/models/module_clip.py

@@ -0,0 +1,538 @@
+# The implementation is  adopated from the CLIP4Clip implementation,
+# made pubicly available under Apache License, Version 2.0 at https://github.com/ArrowLuo/CLIP4Clip
+
+import hashlib
+import os
+import urllib
+import warnings
+from collections import OrderedDict
+from typing import Tuple, Union
+
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint as checkpoint
+from torch import nn
+from tqdm import tqdm
+
+_MODELS = {}
+_PT_NAME = {'ViT-B/16': 'ViT-B-16.pt'}
+
+
+def available_models():
+    """Returns the names of available CLIP models"""
+    return list(_MODELS.keys())
+
+
+class Bottleneck(nn.Module):
+    expansion = 4
+
+    def __init__(self, inplanes, planes, stride=1):
+        super(Bottleneck, self).__init__()
+
+        # all conv layers have stride 1. an avgpool is performed after the second convolution when stride > 1
+        self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False)
+        self.bn1 = nn.BatchNorm2d(planes)
+
+        self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False)
+        self.bn2 = nn.BatchNorm2d(planes)
+
+        self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity()
+
+        self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False)
+        self.bn3 = nn.BatchNorm2d(planes * self.expansion)
+
+        self.relu = nn.ReLU(inplace=True)
+        self.downsample = None
+        self.stride = stride
+
+        if stride > 1 or inplanes != planes * Bottleneck.expansion:
+            # downsampling layer is prepended with an avgpool, and the subsequent convolution has stride 1
+            self.downsample = nn.Sequential(
+                OrderedDict([('-1', nn.AvgPool2d(stride)),
+                             ('0',
+                              nn.Conv2d(
+                                  inplanes,
+                                  planes * self.expansion,
+                                  1,
+                                  stride=1,
+                                  bias=False)),
+                             ('1', nn.BatchNorm2d(planes * self.expansion))]))
+
+    def forward(self, x: torch.Tensor):
+        identity = x
+
+        out = self.relu(self.bn1(self.conv1(x)))
+        out = self.relu(self.bn2(self.conv2(out)))
+        out = self.avgpool(out)
+        out = self.bn3(self.conv3(out))
+
+        if self.downsample is not None:
+            identity = self.downsample(x)
+
+        out += identity
+        out = self.relu(out)
+        return out
+
+
+class AttentionPool2d(nn.Module):
+
+    def __init__(self,
+                 spacial_dim: int,
+                 embed_dim: int,
+                 num_heads: int,
+                 output_dim: int = None):
+        super(AttentionPool2d, self).__init__()
+        self.positional_embedding = nn.Parameter(
+            torch.randn(spacial_dim**2 + 1, embed_dim) / embed_dim**0.5)
+        self.k_proj = nn.Linear(embed_dim, embed_dim)
+        self.q_proj = nn.Linear(embed_dim, embed_dim)
+        self.v_proj = nn.Linear(embed_dim, embed_dim)
+        self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim)
+        self.num_heads = num_heads
+
+    def forward(self, x):
+        x = x.reshape(x.shape[0], x.shape[1],
+                      x.shape[2] * x.shape[3]).permute(2, 0,
+                                                       1)  # NCHW -> (HW)NC
+        x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0)  # (HW+1)NC
+        x = x + self.positional_embedding[:, None, :].to(x.dtype)  # (HW+1)NC
+        x, _ = F.multi_head_attention_forward(
+            query=x,
+            key=x,
+            value=x,
+            embed_dim_to_check=x.shape[-1],
+            num_heads=self.num_heads,
+            q_proj_weight=self.q_proj.weight,
+            k_proj_weight=self.k_proj.weight,
+            v_proj_weight=self.v_proj.weight,
+            in_proj_weight=None,
+            in_proj_bias=torch.cat(
+                [self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]),
+            bias_k=None,
+            bias_v=None,
+            add_zero_attn=False,
+            dropout_p=0,
+            out_proj_weight=self.c_proj.weight,
+            out_proj_bias=self.c_proj.bias,
+            use_separate_proj_weight=True,
+            training=self.training,
+            need_weights=False)
+
+        return x[0]
+
+
+class ModifiedResNet(nn.Module):
+    """
+    A ResNet class that is similar to torchvision's but contains the following changes:
+    - There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool.
+    - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1
+    - The final pooling layer is a QKV attention instead of an average pool
+    """
+
+    def __init__(self,
+                 layers,
+                 output_dim,
+                 heads,
+                 input_resolution=224,
+                 width=64):
+        super(ModifiedResNet, self).__init__()
+        self.output_dim = output_dim
+        self.input_resolution = input_resolution
+
+        # the 3-layer stem
+        self.conv1 = nn.Conv2d(
+            3, width // 2, kernel_size=3, stride=2, padding=1, bias=False)
+        self.bn1 = nn.BatchNorm2d(width // 2)
+        self.conv2 = nn.Conv2d(
+            width // 2, width // 2, kernel_size=3, padding=1, bias=False)
+        self.bn2 = nn.BatchNorm2d(width // 2)
+        self.conv3 = nn.Conv2d(
+            width // 2, width, kernel_size=3, padding=1, bias=False)
+        self.bn3 = nn.BatchNorm2d(width)
+        self.avgpool = nn.AvgPool2d(2)
+        self.relu = nn.ReLU(inplace=True)
+
+        # residual layers
+        self._inplanes = width  # this is a *mutable* variable used during construction
+        self.layer1 = self._make_layer(width, layers[0])
+        self.layer2 = self._make_layer(width * 2, layers[1], stride=2)
+        self.layer3 = self._make_layer(width * 4, layers[2], stride=2)
+        self.layer4 = self._make_layer(width * 8, layers[3], stride=2)
+
+        embed_dim = width * 32  # the ResNet feature dimension
+        self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim,
+                                        heads, output_dim)
+
+    def _make_layer(self, planes, blocks, stride=1):
+        layers = [Bottleneck(self._inplanes, planes, stride)]
+
+        self._inplanes = planes * Bottleneck.expansion
+        for _ in range(1, blocks):
+            layers.append(Bottleneck(self._inplanes, planes))
+
+        return nn.Sequential(*layers)
+
+    def forward(self, x):
+
+        def stem(x):
+            for conv, bn in [(self.conv1, self.bn1), (self.conv2, self.bn2),
+                             (self.conv3, self.bn3)]:
+                x = self.relu(bn(conv(x)))
+            x = self.avgpool(x)
+            return x
+
+        x = x.type(self.conv1.weight.dtype)
+        x = stem(x)
+        x = self.layer1(x)
+        x = self.layer2(x)
+        x = self.layer3(x)
+        x = self.layer4(x)
+        x = self.attnpool(x)
+
+        return x
+
+
+class LayerNorm(nn.LayerNorm):
+    """Subclass torch's LayerNorm to handle fp16."""
+
+    def forward(self, x: torch.Tensor):
+        orig_type = x.dtype
+        ret = super().forward(x.type(torch.float32))
+        return ret.type(orig_type)
+
+
+class QuickGELU(nn.Module):
+
+    def forward(self, x: torch.Tensor):
+        return x * torch.sigmoid(1.702 * x)
+
+
+class ResidualAttentionBlock(nn.Module):
+
+    def __init__(self, d_model: int, n_head: int, attn_mask=None):
+        super(ResidualAttentionBlock, self).__init__()
+
+        self.attn = nn.MultiheadAttention(d_model, n_head)
+        self.ln_1 = LayerNorm(d_model)
+        self.mlp = nn.Sequential(
+            OrderedDict([('c_fc', nn.Linear(d_model, d_model * 4)),
+                         ('gelu', QuickGELU()),
+                         ('c_proj', nn.Linear(d_model * 4, d_model))]))
+        self.ln_2 = LayerNorm(d_model)
+        self.attn_mask = attn_mask
+
+    def attention(self, x: torch.Tensor):
+        attn_mask_ = self.attn_mask
+        if self.attn_mask is not None and hasattr(self.attn_mask, '__call__'):
+            attn_mask_ = self.attn_mask(x.size(0))  # LND
+
+        attn_mask_ = attn_mask_.to(
+            dtype=x.dtype, device=x.device) if attn_mask_ is not None else None
+        return self.attn(x, x, x, need_weights=False, attn_mask=attn_mask_)[0]
+
+    def forward(self, x):
+        x = x + self.attention(self.ln_1(x))
+        x = x + self.mlp(self.ln_2(x))
+        return x
+
+
+class Transformer(nn.Module):
+
+    def __init__(self,
+                 width: int,
+                 layers: int,
+                 heads: int,
+                 attn_mask=None,
+                 use_gc=0):
+        super(Transformer, self).__init__()
+        self.width = width
+        self.layers = layers
+        self.resblocks = nn.Sequential(*[
+            ResidualAttentionBlock(width, heads, attn_mask)
+            for _ in range(layers)
+        ])
+
+        self.use_gc = use_gc
+
+    def forward(self, x: torch.Tensor):
+        if self.use_gc > 0:
+            for blk in self.resblocks:
+                x = checkpoint.checkpoint(blk, x)
+            return x
+        else:
+            return self.resblocks(x)
+
+
+class VisualTransformer(nn.Module):
+
+    def __init__(self,
+                 input_resolution: int,
+                 patch_size: int,
+                 width: int,
+                 layers: int,
+                 heads: int,
+                 output_dim: int,
+                 linear_patch: str = '2d',
+                 use_gc: int = 0):
+        super(VisualTransformer, self).__init__()
+        self.input_resolution = input_resolution
+        self.output_dim = output_dim
+
+        self.conv1 = nn.Conv2d(
+            in_channels=3,
+            out_channels=width,
+            kernel_size=patch_size,
+            stride=patch_size,
+            bias=False)
+
+        scale = width**-0.5
+        self.class_embedding = nn.Parameter(scale * torch.randn(width))
+        self.positional_embedding = nn.Parameter(scale * torch.randn(
+            (input_resolution // patch_size)**2 + 1, width))
+        self.ln_pre = LayerNorm(width)
+
+        self.transformer = Transformer(width, layers, heads, use_gc=use_gc)
+
+        self.ln_post = LayerNorm(width)
+        self.proj = nn.Parameter(scale * torch.randn(width, output_dim))
+
+        # For 3D
+        assert linear_patch in ['2d', '3d']
+        self.linear_patch = linear_patch
+        if self.linear_patch == '3d':
+            self.conv2 = nn.Conv3d(
+                in_channels=3,
+                out_channels=width,
+                kernel_size=(3, patch_size, patch_size),
+                stride=(1, patch_size, patch_size),
+                padding=(1, 0, 0),
+                bias=False)
+
+    def forward(self, x: torch.Tensor, video_frame=-1):
+
+        if self.linear_patch == '3d':
+            assert video_frame != -1
+            x_3d = x.reshape(-1, video_frame, x.shape[-3], x.shape[-2],
+                             x.shape[-1])
+            x_3d = x_3d.permute(0, 2, 1, 3, 4)
+            x_3d = self.conv2(x_3d)  # shape = [*, width, frame, grid, grid]
+            x_3d = x_3d.permute(0, 2, 1, 3,
+                                4)  # shape = [*, frame, width, grid, grid]
+            x = x_3d.reshape(
+                -1, x_3d.shape[-3], x_3d.shape[-2],
+                x_3d.shape[-1]).contiguous()  # shape = [*, width, grid, grid]
+        else:
+            x = self.conv1(x)  # shape = [*, width, grid, grid]
+
+        x = x.reshape(x.shape[0], x.shape[1],
+                      -1)  # shape = [*, width, grid ** 2]
+        x = x.permute(0, 2, 1)  # shape = [*, grid ** 2, width]
+
+        _x = self.class_embedding.to(x.dtype) + torch.zeros(
+            x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device)
+        x = torch.cat([_x, x], dim=1)
+        x = x + self.positional_embedding.to(x.dtype)
+        x = self.ln_pre(x)
+
+        x = x.permute(1, 0, 2)  # NLD -> LND
+        x = self.transformer(x)
+        x = x.permute(1, 0, 2)  # LND -> NLD
+
+        return x
+
+
+class CLIP(nn.Module):
+
+    def __init__(
+            self,
+            embed_dim: int,
+            # vision
+            image_resolution: int,
+            vision_layers: Union[Tuple[int, int, int, int], int],
+            vision_width: int,
+            vision_patch_size: int,
+            # text
+            context_length: int,
+            vocab_size: int,
+            transformer_width: int,
+            transformer_heads: int,
+            transformer_layers: int,
+            # vision linear of patch
+            linear_patch: str = '2d',
+            use_gc: int = 0):
+        super(CLIP, self).__init__()
+
+        self.context_length = context_length
+
+        if isinstance(vision_layers, (tuple, list)):
+            vision_heads = vision_width * 32 // 64
+            self.visual = ModifiedResNet(
+                layers=vision_layers,
+                output_dim=embed_dim,
+                heads=vision_heads,
+                input_resolution=image_resolution,
+                width=vision_width)
+        else:
+            vision_heads = vision_width // 64
+            self.visual = VisualTransformer(
+                input_resolution=image_resolution,
+                patch_size=vision_patch_size,
+                width=vision_width,
+                layers=vision_layers,
+                heads=vision_heads,
+                output_dim=embed_dim,
+                linear_patch=linear_patch,
+                use_gc=use_gc)
+
+        self.transformer = Transformer(
+            width=transformer_width,
+            layers=transformer_layers,
+            heads=transformer_heads,
+            attn_mask=self.build_attention_mask)
+
+        self.vocab_size = vocab_size
+        self.token_embedding = nn.Embedding(vocab_size, transformer_width)
+        self.positional_embedding = nn.Parameter(
+            torch.empty(self.context_length, transformer_width))
+        self.ln_final = LayerNorm(transformer_width)
+
+        self.text_projection = nn.Parameter(
+            torch.empty(transformer_width, embed_dim))
+        self.logit_scale = nn.Parameter(torch.ones([]))
+
+        self.initialize_parameters()
+
+    def initialize_parameters(self):
+        nn.init.normal_(self.token_embedding.weight, std=0.02)
+        nn.init.normal_(self.positional_embedding, std=0.01)
+
+        if isinstance(self.visual, ModifiedResNet):
+            if self.visual.attnpool is not None:
+                std = self.visual.attnpool.c_proj.in_features**-0.5
+                nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std)
+                nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std)
+                nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std)
+                nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std)
+
+            for resnet_block in [
+                    self.visual.layer1, self.visual.layer2, self.visual.layer3,
+                    self.visual.layer4
+            ]:
+                for name, param in resnet_block.named_parameters():
+                    if name.endswith('bn3.weight'):
+                        nn.init.zeros_(param)
+
+        proj_std = (self.transformer.width**-0.5) * (
+            (2 * self.transformer.layers)**-0.5)
+        attn_std = self.transformer.width**-0.5
+        fc_std = (2 * self.transformer.width)**-0.5
+        for block in self.transformer.resblocks:
+            nn.init.normal_(block.attn.in_proj_weight, std=attn_std)
+            nn.init.normal_(block.attn.out_proj.weight, std=proj_std)
+            nn.init.normal_(block.mlp.c_fc.weight, std=fc_std)
+            nn.init.normal_(block.mlp.c_proj.weight, std=proj_std)
+
+        if self.text_projection is not None:
+            nn.init.normal_(
+                self.text_projection, std=self.transformer.width**-0.5)
+
+    def build_attention_mask(self, context_length):
+        # lazily create causal attention mask, with full attention between the vision tokens
+        # pytorch uses additive attention mask; fill with -inf
+        mask = torch.zeros(context_length, context_length)
+        mask.fill_(float('-inf'))
+        mask.triu_(1)  # zero out the lower diagonal
+        return mask
+
+    @staticmethod
+    def get_config(model_dir):
+        model_path = '{}/ViT-B-16.pt'.format(model_dir)
+        try:
+            # loading JIT archive
+            model = torch.jit.load(model_path, map_location='cpu').eval()
+            state_dict = model.state_dict()
+        except RuntimeError:
+            state_dict = torch.load(model_path, map_location='cpu')
+        return state_dict
+
+    @property
+    def dtype(self):
+        return self.visual.conv1.weight.dtype
+
+    def encode_image_tokens(self, image, return_hidden=False):
+        hidden = self.visual(image.type(self.dtype))
+        hidden = self.visual.ln_post(hidden) @ self.visual.proj
+
+        x = hidden[:, 0, :]
+
+        if return_hidden:
+            return x, hidden
+
+        return x
+
+    def encode_text(self, text, return_hidden=False, prompt=None):
+        x = self.token_embedding(text).type(
+            self.dtype)  # [batch_size, n_ctx, d_model]
+        if prompt:
+            x = prompt(x)
+
+        pos_emd = self.positional_embedding[:x.size(1), :].type(self.dtype)
+        x = x + pos_emd
+        x = x.permute(1, 0, 2)  # NLD -> LND
+        x = self.transformer(x)
+        x = x.permute(1, 0, 2)  # LND -> NLD
+
+        hidden = self.ln_final(x).type(self.dtype) @ self.text_projection
+
+        # take features from the eot embedding (eot_token is the highest number in each sequence)
+        x = hidden[torch.arange(hidden.shape[0]), text.argmax(dim=-1)]
+
+        if return_hidden:
+            return x, hidden
+
+        return x
+
+    def forward(self, image, text):
+        image_features = self.encode_image(image)
+        text_features = self.encode_text(text)
+
+        # normalized features
+        image_features = image_features / image_features.norm(
+            dim=-1, keepdim=True)
+        text_features = text_features / text_features.norm(
+            dim=-1, keepdim=True)
+
+        # cosine similarity as logits
+        logit_scale = self.logit_scale.exp()
+        logits_per_image = logit_scale * image_features @ text_features.t()
+        logits_per_text = logit_scale * text_features @ image_features.t()
+
+        return logits_per_image, logits_per_text
+
+
+def convert_weights(model: nn.Module):
+    """Convert applicable model parameters to fp16"""
+
+    def _convert_weights_to_fp16(lay):
+        # l = lay
+        if isinstance(lay, (nn.Conv1d, nn.Conv2d, nn.Conv3d, nn.Linear)):
+            lay.weight.data = lay.weight.data.half()
+            if lay.bias is not None:
+                lay.bias.data = lay.bias.data.half()
+
+        if isinstance(lay, nn.MultiheadAttention):
+            for attr in [
+                    *[f'{s}_proj_weight' for s in ['in', 'q', 'k', 'v']],
+                    'in_proj_bias', 'bias_k', 'bias_v'
+            ]:
+                tensor = getattr(lay, attr)
+                if tensor is not None:
+                    tensor.data = tensor.data.half()
+
+        for name in ['text_projection', 'proj']:
+            if hasattr(lay, name):
+                attr = getattr(lay, name)
+                if attr is not None:
+                    attr.data = attr.data.half()
+
+    model.apply(_convert_weights_to_fp16)

modelscope/models/multi_modal/prost/models/module_cross.py

@@ -0,0 +1,249 @@
+from __future__ import absolute_import, division, print_function
+import copy
+import logging
+import math
+import os
+import shutil
+import tarfile
+import tempfile
+from collections import OrderedDict
+
+import json
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+from .until_config import PreCrossConfig
+from .until_module import ACT2FN, LayerNorm, PreTrainedModel
+
+
+# PRETRAINED_MODEL_ARCHIVE_MAP = {}
+# CONFIG_NAME = 'cross_config.json'
+# WEIGHTS_NAME = 'cross_pytorch_model.bin'
+class CrossConfig(PreCrossConfig):
+    """Configuration class to store the configuration of a `CrossModel`.
+    """
+
+    # pretrained_model_archive_map = PRETRAINED_MODEL_ARCHIVE_MAP
+    # config_name = CONFIG_NAME
+    # weights_name = WEIGHTS_NAME
+    def __init__(self,
+                 vocab_size_or_config_json_file,
+                 hidden_size=768,
+                 num_hidden_layers=12,
+                 num_attention_heads=12,
+                 intermediate_size=3072,
+                 hidden_act='gelu',
+                 hidden_dropout_prob=0.1,
+                 attention_probs_dropout_prob=0.1,
+                 max_position_embeddings=512,
+                 type_vocab_size=2,
+                 initializer_range=0.02):
+        """Constructs CrossConfig.
+
+        Args:
+            vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `CrossModel`.
+            hidden_size: Size of the encoder layers and the pooler layer.
+            num_hidden_layers: Number of hidden layers in the Transformer encoder.
+            num_attention_heads: Number of attention heads for each attention layer in
+                the Transformer encoder.
+            intermediate_size: The size of the "intermediate" (i.e., feed-forward)
+                layer in the Transformer encoder.
+            hidden_act: The non-linear activation function (function or string) in the
+                encoder and pooler. If string, "gelu", "relu" and "swish" are supported.
+            hidden_dropout_prob: The dropout probabilitiy for all fully connected
+                layers in the embeddings, encoder, and pooler.
+            attention_probs_dropout_prob: The dropout ratio for the attention
+                probabilities.
+            max_position_embeddings: The maximum sequence length that this model might
+                ever be used with. Typically set this to something large just in case
+                (e.g., 512 or 1024 or 2048).
+            type_vocab_size: The vocabulary size of the `token_type_ids` passed into
+                `CrossModel`.
+            initializer_range: The sttdev of the truncated_normal_initializer for
+                initializing all weight matrices.
+        """
+        if isinstance(vocab_size_or_config_json_file, str):
+            with open(
+                    vocab_size_or_config_json_file, 'r',
+                    encoding='utf-8') as reader:
+                json_config = json.loads(reader.read())
+            for key, value in json_config.items():
+                self.__dict__[key] = value
+        elif isinstance(vocab_size_or_config_json_file, int):
+            self.vocab_size = vocab_size_or_config_json_file
+            self.hidden_size = hidden_size
+            self.num_hidden_layers = num_hidden_layers
+            self.num_attention_heads = num_attention_heads
+            self.hidden_act = hidden_act
+            self.intermediate_size = intermediate_size
+            self.hidden_dropout_prob = hidden_dropout_prob
+            self.attention_probs_dropout_prob = attention_probs_dropout_prob
+            self.max_position_embeddings = max_position_embeddings
+            self.type_vocab_size = type_vocab_size
+            self.initializer_range = initializer_range
+        else:
+            raise ValueError(
+                'First argument must be either a vocabulary size (int)'
+                'or the path to a pretrained model config file (str)')
+
+
+class QuickGELU(nn.Module):
+
+    def forward(self, x: torch.Tensor):
+        return x * torch.sigmoid(1.702 * x)
+
+
+class ResidualAttentionBlock(nn.Module):
+
+    def __init__(self, d_model: int, n_head: int):
+        super().__init__()
+
+        self.attn = nn.MultiheadAttention(d_model, n_head)
+        self.ln_1 = LayerNorm(d_model)
+        self.mlp = nn.Sequential(
+            OrderedDict([('c_fc', nn.Linear(d_model, d_model * 4)),
+                         ('gelu', QuickGELU()),
+                         ('c_proj', nn.Linear(d_model * 4, d_model))]))
+        self.ln_2 = LayerNorm(d_model)
+        self.n_head = n_head
+
+    def attention(self, x: torch.Tensor, attn_mask: torch.Tensor):
+        attn_mask_ = attn_mask.repeat_interleave(self.n_head, dim=0)
+        return self.attn(x, x, x, need_weights=False, attn_mask=attn_mask_)[0]
+
+    def forward(self, para_tuple: tuple):
+        # x: torch.Tensor, attn_mask: torch.Tensor
+        # print(para_tuple)
+        x, attn_mask = para_tuple
+        x = x + self.attention(self.ln_1(x), attn_mask)
+        x = x + self.mlp(self.ln_2(x))
+        return (x, attn_mask)
+
+
+class Transformer(nn.Module):
+
+    def __init__(self, width: int, layers: int, heads: int):
+        super().__init__()
+        self.width = width
+        self.layers = layers
+        self.resblocks = nn.Sequential(
+            *[ResidualAttentionBlock(width, heads) for _ in range(layers)])
+
+    def forward(self, x: torch.Tensor, attn_mask: torch.Tensor):
+        return self.resblocks((x, attn_mask))[0]
+
+
+class CrossEmbeddings(nn.Module):
+    """Construct the embeddings from word, position and token_type embeddings.
+    """
+
+    def __init__(self, config):
+        super(CrossEmbeddings, self).__init__()
+
+        self.position_embeddings = nn.Embedding(config.max_position_embeddings,
+                                                config.hidden_size)
+        # self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size)
+        # self.LayerNorm = LayerNorm(config.hidden_size, eps=1e-12)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+
+    def forward(self, concat_embeddings, concat_type=None):
+
+        _, seq_length = concat_embeddings.size(0), concat_embeddings.size(1)
+        # if concat_type is None:
+        #     concat_type = torch.zeros(batch_size, concat_type).to(concat_embeddings.device)
+
+        position_ids = torch.arange(
+            seq_length, dtype=torch.long, device=concat_embeddings.device)
+        position_ids = position_ids.unsqueeze(0).expand(
+            concat_embeddings.size(0), -1)
+
+        # token_type_embeddings = self.token_type_embeddings(concat_type)
+        position_embeddings = self.position_embeddings(position_ids)
+
+        embeddings = concat_embeddings + position_embeddings  # + token_type_embeddings
+        # embeddings = self.LayerNorm(embeddings)
+        embeddings = self.dropout(embeddings)
+        return embeddings
+
+
+class CrossPooler(nn.Module):
+
+    def __init__(self, config):
+        super(CrossPooler, self).__init__()
+        self.ln_pool = LayerNorm(config.hidden_size)
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.activation = QuickGELU()
+
+    def forward(self, hidden_states, hidden_mask):
+        # We "pool" the model by simply taking the hidden state corresponding
+        # to the first token.
+        hidden_states = self.ln_pool(hidden_states)
+        pooled_output = hidden_states[:, 0]
+        pooled_output = self.dense(pooled_output)
+        pooled_output = self.activation(pooled_output)
+        return pooled_output
+
+
+class CrossModel(PreTrainedModel):
+
+    def initialize_parameters(self):
+        proj_std = (self.transformer.width**-0.5) * (
+            (2 * self.transformer.layers)**-0.5)
+        attn_std = self.transformer.width**-0.5
+        fc_std = (2 * self.transformer.width)**-0.5
+        for block in self.transformer.resblocks:
+            nn.init.normal_(block.attn.in_proj_weight, std=attn_std)
+            nn.init.normal_(block.attn.out_proj.weight, std=proj_std)
+            nn.init.normal_(block.mlp.c_fc.weight, std=fc_std)
+            nn.init.normal_(block.mlp.c_proj.weight, std=proj_std)
+
+    def __init__(self, config):
+        super(CrossModel, self).__init__(config)
+
+        self.embeddings = CrossEmbeddings(config)
+
+        transformer_width = config.hidden_size
+        transformer_layers = config.num_hidden_layers
+        transformer_heads = config.num_attention_heads
+        self.transformer = Transformer(
+            width=transformer_width,
+            layers=transformer_layers,
+            heads=transformer_heads,
+        )
+        self.pooler = CrossPooler(config)
+        self.apply(self.init_weights)
+
+    def build_attention_mask(self, attention_mask):
+        extended_attention_mask = attention_mask.unsqueeze(1)
+        extended_attention_mask = extended_attention_mask.to(
+            dtype=self.dtype)  # fp16 compatibility
+        extended_attention_mask = (1.0 - extended_attention_mask) * -1000000.0
+        extended_attention_mask = extended_attention_mask.expand(
+            -1, attention_mask.size(1), -1)
+        return extended_attention_mask
+
+    def forward(self,
+                concat_input,
+                concat_type=None,
+                attention_mask=None,
+                output_all_encoded_layers=True):
+
+        if attention_mask is None:
+            attention_mask = torch.ones(
+                concat_input.size(0), concat_input.size(1))
+        if concat_type is None:
+            concat_type = torch.zeros_like(attention_mask)
+
+        extended_attention_mask = self.build_attention_mask(attention_mask)
+
+        embedding_output = self.embeddings(concat_input, concat_type)
+        embedding_output = embedding_output.permute(1, 0, 2)  # NLD -> LND
+        embedding_output = self.transformer(embedding_output,
+                                            extended_attention_mask)
+        embedding_output = embedding_output.permute(1, 0, 2)  # LND -> NLD
+
+        pooled_output = self.pooler(
+            embedding_output, hidden_mask=attention_mask)
+
+        return embedding_output, pooled_output

modelscope/models/multi_modal/prost/models/prost_model.py

@@ -0,0 +1,267 @@
+# The implementation is adopted from the CLIP4Clip implementation,
+# made pubicly available under Apache License, Version 2.0 at https://github.com/ArrowLuo/CLIP4Clip
+
+import os
+import random
+import uuid
+from os.path import exists
+from tempfile import TemporaryDirectory
+from typing import Any, Dict
+from urllib.parse import urlparse
+
+import json
+import numpy as np
+import torch
+from decord import VideoReader, cpu
+from PIL import Image
+
+from modelscope.hub.file_download import http_get_file
+from modelscope.metainfo import Models
+from modelscope.models import TorchModel
+from modelscope.models.builder import MODELS
+from modelscope.models.multi_modal.prost.models.modeling import CLIP4Clip
+from modelscope.models.multi_modal.prost.models.tokenization_clip import \
+    SimpleTokenizer as ClipTokenizer
+from modelscope.utils.constant import ModelFile, Tasks
+from modelscope.utils.logger import get_logger
+from ..dataloaders.rawvideo_util import RawVideoExtractor
+
+logger = get_logger()
+
+
+@MODELS.register_module(Tasks.text_video_retrieval, module_name=Models.prost)
+class ProSTForTVRetrieval(TorchModel):
+
+    def __init__(self, model_dir, **kwargs):
+        super().__init__(model_dir=model_dir, **kwargs)
+        # model config parameters
+        with open(
+                f'{model_dir}/{ModelFile.CONFIGURATION}', 'r',
+                encoding='utf-8') as json_file:
+            all_model_config = json.load(json_file)
+        model_config = all_model_config['paras']
+
+        cross_model_config = all_model_config['crossbase']
+        # print(all_model_config)
+        # print(cross_model_config)
+        model_config['model_dir'] = model_dir
+        self.SPECIAL_TOKEN = {
+            'CLS_TOKEN': '<|startoftext|>',
+            'SEP_TOKEN': '<|endoftext|>',
+            'MASK_TOKEN': '[MASK]',
+            'UNK_TOKEN': '[UNK]',
+            'PAD_TOKEN': '[PAD]'
+        }
+        self.max_words = model_config['max_words']
+        self.max_frames = model_config['max_frames']
+        self.feature_framerate = model_config['feature_framerate']
+        self.image_resolution = 224
+        if torch.cuda.is_available():
+            self.device = model_config['device']
+        else:
+            self.device = 'cpu'
+        self.init_model = f'{model_dir}/{ModelFile.TORCH_MODEL_BIN_FILE}'
+
+        self.tokenizer = ClipTokenizer(model_dir)
+        self.rawVideoExtractor = RawVideoExtractor(
+            frame_rate=self.feature_framerate, size=self.image_resolution)
+        self.local_transform = self.rawVideoExtractor.transform
+        self.model = CLIP4Clip.from_pretrained(
+            cross_config=cross_model_config, task_config=model_config)
+        if hasattr(self.model, 'module'):
+            self.model = self.model.module.to(self.device)
+        else:
+            self.model = self.model.to(self.device)
+        if self.init_model:
+            assert exists(self.init_model)
+            model_state_dict = torch.load(self.init_model, map_location='cpu')
+            self.model.load_state_dict(model_state_dict, strict=False)
+        self.model.to(self.device)
+
+    def _get_text(self, caption, tokenizer, enable_zh=False):
+
+        if type(caption) is str:
+            _caption_text, s, e = caption, None, None
+        elif type(caption) is tuple:
+            if len(caption) == 3:
+                _caption_text, s, e = caption
+            elif len(caption) == 4:
+                _caption_text, s, e, pos = caption
+            else:
+                NotImplementedError
+
+        if isinstance(_caption_text, list):
+            caption_text = random.choice(_caption_text)
+        else:
+            caption_text = _caption_text
+        if enable_zh:
+            _token = tokenizer.encode(caption_text)
+            input_ids = _token.ids
+            input_mask = _token.attention_mask
+            segment_ids = _token.type_ids
+        else:
+            words = tokenizer.tokenize(caption_text)
+
+            words = [self.SPECIAL_TOKEN['CLS_TOKEN']] + words
+            total_length_with_CLS = self.max_words - 1
+            if len(words) > total_length_with_CLS:
+                words = words[:total_length_with_CLS]
+            words = words + [self.SPECIAL_TOKEN['SEP_TOKEN']]
+
+            input_ids = tokenizer.convert_tokens_to_ids(words)
+            input_mask = [1] * len(input_ids)
+            segment_ids = [0] * len(input_ids)
+
+        while len(input_ids) < self.max_words:
+            input_ids.append(0)
+            input_mask.append(0)
+            segment_ids.append(0)
+        assert len(input_ids) == self.max_words
+        assert len(input_mask) == self.max_words
+        assert len(segment_ids) == self.max_words
+
+        pairs_text = np.array(input_ids)
+        pairs_mask = np.array(input_mask)
+        pairs_segment = np.array(segment_ids)
+
+        return pairs_text, pairs_mask, pairs_segment, s, e
+
+    def _get_rawvideo_dec(self,
+                          video_path,
+                          rawVideoExtractor,
+                          local_transform,
+                          s=None,
+                          e=None):
+        video_mask = np.zeros(self.max_frames, dtype=int)
+        max_video_length = 0
+
+        # T x 3 x H x W
+        video = np.zeros((self.max_frames, 3, rawVideoExtractor.size,
+                          rawVideoExtractor.size),
+                         dtype=float)
+
+        if s is None:
+            start_time, end_time = None, None
+        else:
+            start_time = int(s)
+            end_time = int(e)
+            start_time = start_time if start_time >= 0. else 0.
+            end_time = end_time if end_time >= 0. else 0.
+            if start_time > end_time:
+                start_time, end_time = end_time, start_time
+            elif start_time == end_time:
+                end_time = end_time + 1
+
+        url_parsed = urlparse(video_path)
+        if url_parsed.scheme in ('file', '') and exists(
+                url_parsed.path):  # Possibly a local file
+            vreader = VideoReader(video_path, ctx=cpu(0))
+        else:
+            try:
+                with TemporaryDirectory() as temporary_cache_dir:
+                    random_str = uuid.uuid4().hex
+                    http_get_file(
+                        url=video_path,
+                        local_dir=temporary_cache_dir,
+                        file_name=random_str,
+                        cookies=None)
+                    temp_file_path = os.path.join(temporary_cache_dir,
+                                                  random_str)
+                    vreader = VideoReader(temp_file_path, ctx=cpu(0))
+            except Exception as ex:
+                logger.error('non video input, output is {}!!!'.format(ex))
+                return video, video_mask
+
+        fps = vreader.get_avg_fps()
+        f_start = 0 if start_time is None else int(start_time * fps)
+        f_end = int(
+            min(1000000000 if end_time is None else end_time * fps,
+                len(vreader) - 1))
+        num_frames = f_end - f_start + 1
+        if num_frames > 0:
+            # L x T x 3 x H x W
+            sample_fps = int(self.feature_framerate)
+            t_stride = int(round(float(fps) / sample_fps))
+
+            all_pos = list(range(f_start, f_end + 1, t_stride))
+            if len(all_pos) > self.max_frames:
+                sample_pos = [
+                    all_pos[_] for _ in np.linspace(
+                        0, len(all_pos) - 1, num=self.max_frames, dtype=int)
+                ]
+            else:
+                sample_pos = all_pos
+            patch_images = [
+                Image.fromarray(f)
+                for f in vreader.get_batch(sample_pos).asnumpy()
+            ]
+            patch_images = torch.stack(
+                [local_transform(img) for img in patch_images])
+            slice_len = patch_images.shape[0]
+            max_video_length = max_video_length if max_video_length > slice_len else slice_len
+            if slice_len < 1:
+                pass
+            else:
+                video[:slice_len, ...] = patch_images
+        else:
+            logger.error('video path: {} error. video id: {}'.format(
+                video_path, video_id))
+
+        video_mask[:max_video_length] = [1] * max_video_length
+
+        return video, video_mask
+
+    def forward(self, input: Dict[str, Any]) -> Dict[str, Any]:
+
+        from modelscope.outputs import OutputKeys
+        output = {}
+
+        if 'video' in input and input['video'] is not None:
+            video_path = input['video']
+            video, video_mask = self._get_rawvideo_dec(video_path,
+                                                       self.rawVideoExtractor,
+                                                       self.local_transform)
+            video = torch.unsqueeze(
+                torch.from_numpy(video), dim=0).to(self.device)
+            video_mask = torch.unsqueeze(
+                torch.from_numpy(video_mask), dim=0).to(self.device)
+
+        if 'text' in input and input['text'] is not None:
+            caption = input['text']
+            pairs_text, pairs_mask, pairs_segment, s, e = self._get_text(
+                caption, self.tokenizer, enable_zh=False)
+            input_ids = torch.unsqueeze(
+                torch.from_numpy(pairs_text), dim=0).to(self.device)
+            input_mask = torch.unsqueeze(
+                torch.from_numpy(pairs_mask), dim=0).to(self.device)
+            segment_ids = torch.unsqueeze(
+                torch.from_numpy(pairs_segment), dim=0).to(self.device)
+
+        phr_feat, sen_feat, obj_feat, eve_feat = self.model.get_sequence_visual_output(
+            input_ids, segment_ids, input_mask, video, video_mask)
+
+        sim_espm, _, sim_oppm, _ = self.model.get_max_similarity_logits(
+            phr_feat,
+            sen_feat,
+            obj_feat,
+            eve_feat,
+            input_mask,
+            video_mask,
+            shaped=True)
+        # logger.info('sim: {}'.format(sim_espm))
+        # logger.info('sim: {}'.format(sim_oppm))
+        sim_tv = sim_espm + 1.5 * sim_oppm
+
+        # logger.info('phrase prototype: {}'.format(phr_feat.shape))
+        # logger.info('sentence prototype: {}'.format(sen_feat.shape))
+        # logger.info('object prototype: {}'.format(obj_feat.shape))
+        # logger.info('event prototype: {}'.format(eve_feat.shape))
+        output[OutputKeys.TEXTVIDEO_SIM] = sim_tv.cpu().detach().numpy()
+        output[OutputKeys.PHRASE_PROTOTYPE] = phr_feat.cpu().detach().numpy()
+        output[OutputKeys.SENTENCE_PROTOTYPE] = sen_feat.cpu().detach().numpy()
+        output[OutputKeys.OBJECT_PROTOTYPE] = obj_feat.cpu().detach().numpy()
+        output[OutputKeys.EVENT_PROTOTYPE] = eve_feat.cpu().detach().numpy()
+        return output
+
+    def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        return inputs

modelscope/models/multi_modal/prost/models/tokenization_clip.py

@@ -0,0 +1,161 @@
+# The implementation is adopted from the CLIP4Clip implementation,
+# made pubicly available under Apache License, Version 2.0 at https://github.com/ArrowLuo/CLIP4Clip
+
+import gzip
+import html
+import os
+from functools import lru_cache
+
+import ftfy
+import regex as re
+
+
+@lru_cache()
+def bytes_to_unicode():
+    """
+    Returns list of utf-8 byte and a corresponding list of unicode strings.
+    The reversible bpe codes work on unicode strings.
+    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
+    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
+    This is a signficant percentage of your normal, say, 32K bpe vocab.
+    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
+    And avoids mapping to whitespace/control characters the bpe code barfs on.
+    """
+    bs = list(range(ord('!'),
+                    ord('~') + 1)) + list(range(
+                        ord('¡'),
+                        ord('¬') + 1)) + list(range(ord('®'),
+                                                    ord('ÿ') + 1))
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8 + n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+
+def get_pairs(word):
+    """Return set of symbol pairs in a word.
+    Word is represented as tuple of symbols (symbols being variable-length strings).
+    """
+    pairs = set()
+    prev_char = word[0]
+    for char in word[1:]:
+        pairs.add((prev_char, char))
+        prev_char = char
+    return pairs
+
+
+def basic_clean(text):
+    text = ftfy.fix_text(text)
+    text = html.unescape(html.unescape(text))
+    return text.strip()
+
+
+def whitespace_clean(text):
+    text = re.sub(r'\s+', ' ', text)
+    text = text.strip()
+    return text
+
+
+class SimpleTokenizer(object):
+
+    def __init__(self, model_dir):
+        bpe_path = '{}/bpe_simple_vocab_16e6.txt.gz'.format(model_dir)
+        self.byte_encoder = bytes_to_unicode()
+        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
+        merges = gzip.open(bpe_path).read().decode('utf-8').split('\n')
+        merges = merges[1:49152 - 256 - 2 + 1]
+        merges = [tuple(merge.split()) for merge in merges]
+        vocab = list(bytes_to_unicode().values())
+        vocab = vocab + [v + '</w>' for v in vocab]
+        for merge in merges:
+            vocab.append(''.join(merge))
+        vocab.extend(['<|startoftext|>', '<|endoftext|>'])
+        self.encoder = dict(zip(vocab, range(len(vocab))))
+        self.decoder = {v: k for k, v in self.encoder.items()}
+        self.bpe_ranks = dict(zip(merges, range(len(merges))))
+        self.cache = {
+            '<|startoftext|>': '<|startoftext|>',
+            '<|endoftext|>': '<|endoftext|>'
+        }
+        self.pat = re.compile(
+            r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""",
+            re.IGNORECASE)
+
+        self.vocab = self.encoder
+
+    def bpe(self, token):
+        if token in self.cache:
+            return self.cache[token]
+        word = tuple(token[:-1]) + (token[-1] + '</w>', )
+        pairs = get_pairs(word)
+
+        if not pairs:
+            return token + '</w>'
+
+        while True:
+            bigram = min(
+                pairs, key=lambda pair: self.bpe_ranks.get(pair, float('inf')))
+            if bigram not in self.bpe_ranks:
+                break
+            first, second = bigram
+            new_word = []
+            i = 0
+            while i < len(word):
+                try:
+                    j = word.index(first, i)
+                    new_word.extend(word[i:j])
+                    i = j
+                except Exception:
+                    new_word.extend(word[i:])
+                    break
+
+                if word[i] == first and i < len(word) - 1 and word[
+                        i + 1] == second:
+                    new_word.append(first + second)
+                    i += 2
+                else:
+                    new_word.append(word[i])
+                    i += 1
+            new_word = tuple(new_word)
+            word = new_word
+            if len(word) == 1:
+                break
+            else:
+                pairs = get_pairs(word)
+        word = ' '.join(word)
+        self.cache[token] = word
+        return word
+
+    def encode(self, text):
+        bpe_tokens = []
+        text = whitespace_clean(basic_clean(text)).lower()
+        for token in re.findall(self.pat, text):
+            token = ''.join(self.byte_encoder[b]
+                            for b in token.encode('utf-8'))
+            bpe_tokens.extend(self.encoder[bpe_token]
+                              for bpe_token in self.bpe(token).split(' '))
+        return bpe_tokens
+
+    def decode(self, tokens):
+        text = ''.join([self.decoder[token] for token in tokens])
+        text = bytearray([self.byte_decoder[c] for c in text]).decode(
+            'utf-8', errors='replace').replace('</w>', ' ')
+        return text
+
+    def tokenize(self, text):
+        tokens = []
+        text = whitespace_clean(basic_clean(text)).lower()
+        for token in re.findall(self.pat, text):
+            token = ''.join(self.byte_encoder[b]
+                            for b in token.encode('utf-8'))
+            tokens.extend(
+                bpe_token for bpe_token in self.bpe(token).split(' '))
+        return tokens
+
+    def convert_tokens_to_ids(self, tokens):
+        return [self.encoder[bpe_token] for bpe_token in tokens]

modelscope/models/multi_modal/prost/models/until_config.py

@@ -0,0 +1,59 @@
+# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch BERT model."""
+
+from __future__ import absolute_import, division, print_function
+import copy
+import logging
+import os
+import shutil
+import tarfile
+import tempfile
+
+import json
+import torch
+
+# from modelscope.utils.logger import get_logger
+# logger = get_logger(__name__)
+
+
+class PreCrossConfig(object):
+
+    @classmethod
+    def from_dict(cls, json_object):
+        """Constructs a `BertConfig` from a Python dictionary of parameters."""
+        config = cls(vocab_size_or_config_json_file=-1)
+        for key, value in json_object.items():
+            config.__dict__[key] = value
+        return config
+
+    @classmethod
+    def from_json_file(cls, json_file):
+        """Constructs a `BertConfig` from a json file of parameters."""
+        with open(json_file, 'r', encoding='utf-8') as reader:
+            text = reader.read()
+        return cls.from_dict(json.loads(text))
+
+    def __repr__(self):
+        return str(self.to_json_string())
+
+    def to_dict(self):
+        """Serializes this instance to a Python dictionary."""
+        output = copy.deepcopy(self.__dict__)
+        return output
+
+    def to_json_string(self):
+        """Serializes this instance to a JSON string."""
+        return json.dumps(self.to_dict(), indent=2, sort_keys=True) + '\n'

modelscope/models/multi_modal/prost/models/until_module.py

@@ -0,0 +1,574 @@
+# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch BERT model."""
+
+import copy
+import logging
+import math
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+from modelscope.models.multi_modal.prost.models.until_config import \
+    PreCrossConfig
+
+
+def gelu(x):
+    """Implementation of the gelu activation function.
+        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
+        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
+    """
+    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
+
+
+def swish(x):
+    return x * torch.sigmoid(x)
+
+
+ACT2FN = {'gelu': gelu, 'relu': torch.nn.functional.relu, 'swish': swish}
+
+
+class LayerNorm(nn.Module):
+
+    def __init__(self, hidden_size, eps=1e-12):
+        """Construct a layernorm module in the TF style (epsilon inside the square root).
+        """
+        super(LayerNorm, self).__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.bias = nn.Parameter(torch.zeros(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, x):
+        u = x.mean(-1, keepdim=True)
+        s = (x - u).pow(2).mean(-1, keepdim=True)
+        x = (x - u) / torch.sqrt(s + self.variance_epsilon)
+        return self.weight * x + self.bias
+
+
+class CrossEn(nn.Module):
+
+    def __init__(self, config=None):
+        super(CrossEn, self).__init__()
+
+    def forward(self, sim_matrix):
+        logpt = F.log_softmax(sim_matrix, dim=-1)
+        logpt = torch.diag(logpt)
+        nce_loss = -logpt
+        sim_loss = nce_loss.mean()
+        return sim_loss
+
+
+class AllGather(torch.autograd.Function):
+    """An autograd function that performs allgather on a tensor."""
+
+    @staticmethod
+    def forward(ctx, tensor, args):
+        if args.world_size == 1:
+            ctx.rank = args.local_rank
+            ctx.batch_size = tensor.shape[0]
+            return tensor
+        else:
+            output = [torch.empty_like(tensor) for _ in range(args.world_size)]
+            torch.distributed.all_gather(output, tensor)
+            ctx.rank = args.local_rank
+            ctx.batch_size = tensor.shape[0]
+            return torch.cat(output, dim=0)
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        return (
+            grad_output[ctx.batch_size * ctx.rank:ctx.batch_size
+                        * (ctx.rank + 1)],
+            None,
+        )
+
+
+class AllGather2(torch.autograd.Function):
+    """An autograd function that performs allgather on a tensor."""
+    # https://github.com/PyTorchLightning/lightning-bolts/blob/8d3fbf7782e3d3937ab8a1775a7092d7567f2933/pl_bolts/models/self_supervised/simclr/simclr_module.py#L20
+    @staticmethod
+    def forward(ctx, tensor, args):
+        if args.world_size == 1:
+            ctx.rank = args.local_rank
+            ctx.batch_size = tensor.shape[0]
+            return tensor
+        else:
+            output = [torch.empty_like(tensor) for _ in range(args.world_size)]
+            torch.distributed.all_gather(output, tensor)
+            ctx.rank = args.local_rank
+            ctx.batch_size = tensor.shape[0]
+            return torch.cat(output, dim=0)
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        grad_input = grad_output.clone()
+        torch.distributed.all_reduce(
+            grad_input, op=torch.distributed.ReduceOp.SUM, async_op=False)
+        return (grad_input[ctx.rank * ctx.batch_size:(ctx.rank + 1)
+                           * ctx.batch_size], None)
+
+
+class PreTrainedModel(nn.Module):
+    """ An abstract class to handle weights initialization and
+        a simple interface for dowloading and loading pretrained models.
+    """
+
+    def __init__(self, config, *inputs, **kwargs):
+        super(PreTrainedModel, self).__init__()
+        if not isinstance(config, PreCrossConfig):
+            raise ValueError(
+                'Parameter config in `{}(config)` should be an instance of class `PreCrossConfig`. '
+                'To create a model from a Google pretrained model use '
+                '`model = {}.from_pretrained(PRETRAINED_MODEL_NAME)`'.format(
+                    self.__class__.__name__, self.__class__.__name__))
+        self.config = config
+
+    def init_weights(self, module):
+        """ Initialize the weights.
+        """
+        if isinstance(module, (nn.Linear, nn.Embedding)):
+            # Slightly different from the TF version which uses truncated_normal for initialization
+            # cf https://github.com/pytorch/pytorch/pull/5617
+            module.weight.data.normal_(
+                mean=0.0, std=self.config.initializer_range)
+        elif isinstance(module, LayerNorm):
+            if 'beta' in dir(module) and 'gamma' in dir(module):
+                module.beta.data.zero_()
+                module.gamma.data.fill_(1.0)
+            else:
+                module.bias.data.zero_()
+                module.weight.data.fill_(1.0)
+        if isinstance(module, nn.Linear) and module.bias is not None:
+            module.bias.data.zero_()
+
+    def resize_token_embeddings(self, new_num_tokens=None):
+        raise NotImplementedError
+
+    @classmethod
+    def init_preweight(cls, model, state_dict, prefix=None, task_config=None):
+        old_keys = []
+        new_keys = []
+        for key in state_dict.keys():
+            new_key = None
+            if 'gamma' in key:
+                new_key = key.replace('gamma', 'weight')
+            if 'beta' in key:
+                new_key = key.replace('beta', 'bias')
+            if new_key:
+                old_keys.append(key)
+                new_keys.append(new_key)
+        for old_key, new_key in zip(old_keys, new_keys):
+            state_dict[new_key] = state_dict.pop(old_key)
+
+        if prefix is not None:
+            old_keys = []
+            new_keys = []
+            for key in state_dict.keys():
+                old_keys.append(key)
+                new_keys.append(prefix + key)
+            for old_key, new_key in zip(old_keys, new_keys):
+                state_dict[new_key] = state_dict.pop(old_key)
+
+        missing_keys = []
+        unexpected_keys = []
+        error_msgs = []
+        # copy state_dict so _load_from_state_dict can modify it
+        metadata = getattr(state_dict, '_metadata', None)
+        state_dict = state_dict.copy()
+        if metadata is not None:
+            state_dict._metadata = metadata
+
+        def load(module, prefix=''):
+            local_metadata = {} if metadata is None else metadata.get(
+                prefix[:-1], {})
+            module._load_from_state_dict(state_dict, prefix, local_metadata,
+                                         True, missing_keys, unexpected_keys,
+                                         error_msgs)
+            for name, child in module._modules.items():
+                if child is not None:
+                    load(child, prefix + name + '.')
+
+        load(model, prefix='')
+
+        # if prefix is None and (task_config is None or task_config.local_rank == 0):
+        #     logger.info("-" * 20)
+        #     if len(missing_keys) > 0:
+        #         logger.info("Weights of {} not initialized from pretrained model: {}"
+        #                     .format(model.__class__.__name__, "\n   " + "\n   ".join(missing_keys)))
+        #     if len(unexpected_keys) > 0:
+        #         logger.info("Weights from pretrained model not used in {}: {}"
+        #                     .format(model.__class__.__name__, "\n   " + "\n   ".join(unexpected_keys)))
+        #     if len(error_msgs) > 0:
+        #         logger.error("Weights from pretrained model cause errors in {}: {}"
+        #                      .format(model.__class__.__name__, "\n   " + "\n   ".join(error_msgs)))
+
+        return model
+
+    @property
+    def dtype(self):
+        """
+        :obj:`torch.dtype`: The dtype of the module (assuming that all the module parameters have the same dtype).
+        """
+        try:
+            return next(self.parameters()).dtype
+        except StopIteration:
+            # For nn.DataParallel compatibility in PyTorch 1.5
+            def find_tensor_attributes(module: nn.Module):
+                tuples = [(k, v) for k, v in module.__dict__.items()
+                          if torch.is_tensor(v)]
+                return tuples
+
+            gen = self._named_members(get_members_fn=find_tensor_attributes)
+            first_tuple = next(gen)
+            return first_tuple[1].dtype
+
+    @classmethod
+    def from_pretrained(cls, config, state_dict=None, *inputs, **kwargs):
+        """
+        Instantiate a PreTrainedModel from a pre-trained model file or a pytorch state dict.
+        Download and cache the pre-trained model file if needed.
+        """
+        # Instantiate model.
+        model = cls(config, *inputs, **kwargs)
+        if state_dict is None:
+            return model
+        model = cls.init_preweight(model, state_dict)
+
+        return model
+
+
+class PatchShiftModule(nn.Module):
+
+    def __init__(self, net, video_frame, n_div):
+        super().__init__()
+        self.net = net
+        self.video_frame = video_frame
+        self.n_div = n_div
+
+    def forward(self,
+                query,
+                key,
+                value,
+                key_padding_mask=None,
+                need_weights=True,
+                attn_mask=None):
+        # here q == k == v, psm means patch shift output
+        x = query  # shape here is LND, not NLD (50, 384, 768)
+        x = x.permute(1, 0, 2)  # LND -> NLD
+        patch_len = x.shape[-2]
+        fold = patch_len // self.n_div
+        x = x.reshape(-1, self.video_frame, x.shape[-2],
+                      x.shape[-1])  # shape = [bs, frame, grid ** 2, width]
+        psm = torch.zeros_like(x)  # shape = [bs, frame, grid ** 2, width]
+        psm[:, :, :, :] = x[:, :, :, :]
+        lshift_indices = torch.arange(start=1, end=patch_len, step=fold)
+        psm[:, 1:, lshift_indices, :] = x[:, :-1,
+                                          lshift_indices, :]  # f_t = f_t-1
+        rshift_indices = torch.arange(start=1 + 3, end=patch_len, step=fold)
+        psm[:, :-1, rshift_indices, :] = x[:, 1:,
+                                           rshift_indices, :]  # f_t = f_t+1
+        x = psm.reshape(-1, patch_len, x.shape[-1])
+        x = x.permute(1, 0, 2)  # NLD -> LND
+
+        return self.net(
+            x, x, x, need_weights=need_weights, attn_mask=attn_mask)
+
+
+def make_patch_shift(net, video_frame=12, shift_layers=4, n_div=7):
+    '''
+    Args:
+    net: CLIP
+    video_frame: need predefine here
+    shift_layers: layers to be shift
+    '''
+
+    def make_trans_patch_shift(stage, shift_layers):
+        blocks = list(stage.children())
+        for i, b in enumerate(blocks):
+            if i >= 10 and i <= 11:
+                blocks[i].attn = PatchShiftModule(
+                    b.attn, video_frame=video_frame, n_div=n_div)
+        return nn.Sequential(*blocks)
+
+    net.clip.visual.transformer.resblocks = make_trans_patch_shift(
+        net.clip.visual.transformer.resblocks, shift_layers=shift_layers)
+
+
+def _get_clones(module, N):
+    return nn.ModuleList([copy.deepcopy(module) for i in range(N)])
+
+
+class Event_Layer(nn.Module):
+
+    def __init__(self,
+                 d_model,
+                 nhead,
+                 dim_feedforward=2048,
+                 dropout=0.1,
+                 activation='relu',
+                 normalize_before=False,
+                 is_weights=False):
+        super().__init__()
+        self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
+        self.self_attn_vis = nn.MultiheadAttention(
+            d_model, nhead, dropout=dropout)
+        self.multihead_attn = nn.MultiheadAttention(
+            d_model, nhead, dropout=dropout)
+        # Implementation of Feedforward model
+        self.linear1 = nn.Linear(d_model, dim_feedforward)
+        self.dropout = nn.Dropout(dropout)
+        self.linear2 = nn.Linear(dim_feedforward, d_model)
+
+        self.norm1 = nn.LayerNorm(d_model)
+        self.norm2 = nn.LayerNorm(d_model)
+        self.norm3 = nn.LayerNorm(d_model)
+        self.norm4 = nn.LayerNorm(d_model)
+        self.norm5 = nn.LayerNorm(d_model)
+
+        self.dropout1 = nn.Dropout(dropout)
+        self.dropout2 = nn.Dropout(dropout)
+        self.dropout3 = nn.Dropout(dropout)
+
+        self.activation = nn.ReLU(inplace=True)
+        self.normalize_before = normalize_before
+        self.is_weights = is_weights
+
+    def forward(self, tgt, memory, pos=None, query_pos=None):
+
+        tgt = self.norm1(tgt)
+        memory = self.norm2(memory)
+        tgt = self.self_attn(tgt, tgt, tgt)[0]
+        tgt = self.norm3(tgt)
+
+        tgt2, atten_weights = self.multihead_attn(tgt, memory, memory)
+        tgt = tgt + self.dropout1(tgt2)
+        tgt = self.norm4(tgt)
+        tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt))))
+        tgt = tgt + self.dropout2(tgt2)
+        tgt = self.norm5(tgt)
+
+        return tgt, atten_weights
+
+
+def adaptive_mask(aa, bb, ada_para):
+    tensor = torch.zeros((aa, bb))
+    adaptive_num = int(bb * ada_para)
+    cc = int(bb / aa)
+    for i in range(aa):
+        start_col = i * cc
+        end_col = start_col + cc + adaptive_num
+        if end_col > bb - 1:
+            tmp = end_col - (bb - 1)
+            start_col = start_col - tmp
+            if start_col < 0:
+                start_col = 0
+            end_col = bb
+        tensor[i, start_col:end_col] = 1
+    tensor = ~tensor.bool()
+    return tensor
+
+
+class Frame_Layer(nn.Module):
+
+    def __init__(self,
+                 d_model,
+                 nhead,
+                 dim_feedforward=2048,
+                 para=1.0,
+                 dropout=0.1,
+                 activation='relu',
+                 normalize_before=False,
+                 is_weights=False):
+        super().__init__()
+        self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
+        self.self_attn_vis = nn.MultiheadAttention(
+            d_model, nhead, dropout=dropout)
+        self.multihead_attn = nn.MultiheadAttention(
+            d_model, nhead, dropout=dropout)
+        # Implementation of Feedforward model
+        self.linear1 = nn.Linear(d_model, dim_feedforward)
+        self.dropout = nn.Dropout(dropout)
+        self.linear2 = nn.Linear(dim_feedforward, d_model)
+
+        self.norm1 = nn.LayerNorm(d_model)
+        self.norm2 = nn.LayerNorm(d_model)
+        self.norm3 = nn.LayerNorm(d_model)
+        self.norm4 = nn.LayerNorm(d_model)
+
+        self.dropout1 = nn.Dropout(dropout)
+        self.dropout2 = nn.Dropout(dropout)
+
+        self.activation = nn.ReLU(inplace=True)
+        self.normalize_before = normalize_before
+        self.is_weights = is_weights
+        self.mask_para = para
+
+    def forward(self, tgt, memory, pos=None, query_pos=None):
+        tgt = self.norm1(tgt)
+        memory = self.norm2(memory)
+        mask_new = adaptive_mask(tgt.shape[0], memory.shape[0], ada_para=0.2)
+        tgt2, atten_weights = self.multihead_attn(
+            tgt, memory, memory, attn_mask=mask_new.cuda())
+        tgt = tgt + self.dropout1(tgt2)
+
+        tgt = self.norm3(tgt)
+        tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt))))
+        tgt = tgt + self.dropout2(tgt2)
+        tgt = self.norm4(tgt)
+
+        return tgt, atten_weights
+
+
+class TransDecoder(nn.Module):
+
+    def __init__(self,
+                 decoder_layer,
+                 num_layers,
+                 norm=None,
+                 return_intermediate=False):
+        super().__init__()
+        self.layers = _get_clones(decoder_layer, num_layers)
+        self.num_layers = num_layers
+        self.norm = norm
+        self.return_intermediate = return_intermediate
+
+    def forward(self, tgt, memory, pos=None, query_pos=None):
+        output = tgt
+
+        intermediate = []
+        all_weights = []
+
+        for layer in self.layers:
+            output, weights = layer(
+                output, memory, pos=pos, query_pos=query_pos)
+            if self.return_intermediate:
+                intermediate.append(self.norm(output))
+                all_weights.append(weights)
+
+        if self.norm is not None:
+            output = self.norm(output)
+            if self.return_intermediate:
+                intermediate.pop()
+                intermediate.append(output)
+
+        if self.return_intermediate:
+            return torch.stack(intermediate), torch.stack(all_weights)
+        return output.unsqueeze(0)
+
+
+class Event_decoder(nn.Module):
+
+    def __init__(self,
+                 num_attris=3,
+                 layers=1,
+                 heads=1,
+                 dim_ftr=512,
+                 pos_emb=False,
+                 length=1,
+                 dim_feedforward=512,
+                 without_init=False):
+        super().__init__()
+        embedding_dim = dim_ftr
+
+        d_model = dim_ftr
+        dim_feedforward = dim_feedforward
+
+        self.V = nn.Parameter(
+            torch.Tensor(num_attris, dim_feedforward), requires_grad=True)
+        nn.init.xavier_uniform_(self.V)
+        decoder_layer = Event_Layer(
+            d_model=d_model, nhead=heads, dim_feedforward=dim_feedforward)
+        self.event_decoder = TransDecoder(
+            decoder_layer,
+            layers,
+            nn.LayerNorm(d_model),
+            return_intermediate=True)
+        self.use_pos_enc = pos_emb
+
+        if self.use_pos_enc:
+            self.position_encoding_pre = positionalencoding2d(
+                embedding_dim, 14, 14).unsqueeze(0)
+
+    def forward(self, features):
+        batch_size = features.shape[0]
+        if self.use_pos_enc:  # False
+            pos_encoding = self.position_encoding_pre(
+                features,
+                torch.zeros(features.shape[0], 14, 14,
+                            dtype=torch.bool).cuda())
+            features = features + pos_encoding
+
+        enco_others = features.permute(1, 0, 2)
+        h_attr = self.V
+        h_attr_batch = h_attr.unsqueeze(0).repeat(batch_size, 1, 1)
+        h_attr_batch = h_attr_batch.permute(1, 0, 2)
+
+        hs, _ = self.event_decoder(h_attr_batch, enco_others)
+        hs = hs[-1].permute(1, 0, 2)
+        return hs
+
+
+class Frame_decoder(nn.Module):
+
+    def __init__(self,
+                 num_attris=3,
+                 layers=1,
+                 heads=1,
+                 dim_ftr=512,
+                 pos_emb=False,
+                 length=1,
+                 dim_feedforward=512,
+                 without_init=False):
+        super().__init__()
+        embedding_dim = dim_ftr
+        d_model = dim_ftr
+        dim_feedforward = dim_feedforward
+
+        self.V = nn.Parameter(
+            torch.Tensor(num_attris, dim_feedforward), requires_grad=True)
+        nn.init.xavier_uniform_(self.V)
+        decoder_layer = Frame_Layer(
+            d_model=d_model, nhead=heads, dim_feedforward=dim_feedforward)
+        self.event_decoder = TransDecoder(
+            decoder_layer,
+            layers,
+            nn.LayerNorm(d_model),
+            return_intermediate=True)
+        self.use_pos_enc = pos_emb
+
+        if self.use_pos_enc:
+            self.position_encoding_pre = positionalencoding2d(
+                embedding_dim, 14, 14).unsqueeze(0)
+
+    def forward(self, features):
+        batch_size = features.shape[0]
+        if self.use_pos_enc:
+            pos_encoding = self.position_encoding_pre(
+                features,
+                torch.zeros(features.shape[0], 14, 14,
+                            dtype=torch.bool).cuda())
+            features = features + pos_encoding
+
+        enco_others = features.permute(1, 0, 2)
+        h_attr = self.V
+        h_attr_batch = h_attr.unsqueeze(0).repeat(batch_size, 1, 1)
+        h_attr_batch = h_attr_batch.permute(1, 0, 2)
+
+        hs, _ = self.event_decoder(h_attr_batch, enco_others)
+        hs = hs[-1].permute(1, 0, 2)
+
+        return hs

modelscope/outputs/outputs.py

@@ -48,6 +48,11 @@ class OutputKeys(object):
     PROBABILITIES = 'probabilities'
     DIALOG_STATES = 'dialog_states'
     VIDEO_EMBEDDING = 'video_embedding'
+    PHRASE_PROTOTYPE = 'phrase_prototype'
+    OBJECT_PROTOTYPE = 'object_prototype'
+    SENTENCE_PROTOTYPE = 'sentence_prototype'
+    EVENT_PROTOTYPE = 'event_prototype'
+    TEXTVIDEO_SIM = 'textvideo_sim'
     UUID = 'uuid'
     WORD = 'word'
     KWS_LIST = 'kws_list'
@@ -106,6 +111,11 @@ OutputTypes = {
     OutputKeys.PROBABILITIES: np.ndarray,
     OutputKeys.DIALOG_STATES: object,
     OutputKeys.VIDEO_EMBEDDING: np.ndarray,
+    OutputKeys.PHRASE_PROTOTYPE: np.ndarray,
+    OutputKeys.OBJECT_PROTOTYPE: np.ndarray,
+    OutputKeys.SENTENCE_PROTOTYPE: np.ndarray,
+    OutputKeys.EVENT_PROTOTYPE: np.ndarray,
+    OutputKeys.TEXTVIDEO_SIM: np.ndarray,
     OutputKeys.UUID: str,
     OutputKeys.WORD: str,
     OutputKeys.KWS_LIST: List[str],
@@ -329,6 +339,24 @@ OutputTypeSchema = {
             'type': 'number'
         }
     },
+    OutputKeys.PHRASE_PROTOTYPE: {
+        'type': 'array',
+        'items': {
+            'type': 'number'
+        }
+    },
+    OutputKeys.OBJECT_PROTOTYPE: {
+        'type': 'array',
+        'items': {
+            'type': 'number'
+        }
+    },
+    OutputKeys.TEXTVIDEO_SIM: {
+        'type': 'array',
+        'items': {
+            'type': 'number'
+        }
+    },
     OutputKeys.UUID: {
         'type': 'string'
     },
@@ -916,6 +944,32 @@ TASK_OUTPUTS = {
     # }
     Tasks.video_embedding: [OutputKeys.VIDEO_EMBEDDING],
 
+    # phrase prototype result for single sentence
+    # {
+    #   "phrase_prototype": np.array with shape [K*D],
+    # }
+    # sentence prototype result for single sentence
+    # {
+    #   "sentence_prototype": np.array with shape [1*D],
+    # }
+    # object prototype result for single video
+    # {
+    #   "object_prototype": np.array with shape [N*K*D],
+    # }
+    # event prototype result for single video
+    # {
+    #   "event_prototype": np.array with shape [N*M*D],
+    # }
+    # text search video result for single sentence
+    # {
+    #   "textvideo_sim": np.array with shape [N*N],
+    # }
+    Tasks.text_video_retrieval: [
+        OutputKeys.PHRASE_PROTOTYPE, OutputKeys.SENTENCE_PROTOTYPE,
+        OutputKeys.OBJECT_PROTOTYPE, OutputKeys.EVENT_PROTOTYPE,
+        OutputKeys.TEXTVIDEO_SIM
+    ],
+
     # video stabilization task result for a single video
     # {"output_video": "path_to_rendered_video"}
     Tasks.video_stabilization: [OutputKeys.OUTPUT_VIDEO],

modelscope/pipeline_inputs.py

@@ -418,6 +418,10 @@ TASK_INPUTS = {
         'img': InputType.IMAGE,
         'text': InputType.TEXT
     },
+    Tasks.text_video_retrieval: {
+        'video': InputType.VIDEO,
+        'text': InputType.TEXT
+    },
     Tasks.visual_question_answering: {
         'image': InputType.IMAGE,
         'text': InputType.TEXT

modelscope/pipelines/multi_modal/__init__.py

@@ -4,30 +4,39 @@ from typing import TYPE_CHECKING
 from modelscope.utils.import_utils import LazyImportModule
 
 if TYPE_CHECKING:
-    from .generative_multi_modal_embedding_pipeline import GEMMMultiModalEmbeddingPipeline
+    from .asr_pipeline import AutomaticSpeechRecognitionPipeline
+    from .diffusers_wrapped import (ChineseStableDiffusionPipeline,
+                                    StableDiffusionPipeline)
+    from .document_vl_embedding_pipeline import DocumentVLEmbeddingPipeline
+    from .generative_multi_modal_embedding_pipeline import \
+        GEMMMultiModalEmbeddingPipeline
     from .image_captioning_pipeline import ImageCaptioningPipeline
-    from .visual_entailment_pipeline import VisualEntailmentPipeline
-    from .visual_grounding_pipeline import VisualGroundingPipeline
+    from .mgeo_ranking_pipeline import MGeoRankingPipeline
     from .multi_modal_embedding_pipeline import MultiModalEmbeddingPipeline
+    from .multimodal_dialogue_pipeline import MultimodalDialoguePipeline
+    from .prost_text_video_retrieval_pipeline import \
+        ProSTTextVideoRetrievalPipeline
+    from .soonet_video_temporal_grounding_pipeline import \
+        SOONetVideoTemporalGroundingPipeline
     from .text_to_image_synthesis_pipeline import TextToImageSynthesisPipeline
+    from .text_to_video_synthesis_pipeline import TextToVideoSynthesisPipeline
+    from .video_captioning_pipeline import VideoCaptioningPipeline
     from .video_multi_modal_embedding_pipeline import \
         VideoMultiModalEmbeddingPipeline
-    from .visual_question_answering_pipeline import VisualQuestionAnsweringPipeline
-    from .asr_pipeline import AutomaticSpeechRecognitionPipeline
-    from .mgeo_ranking_pipeline import MGeoRankingPipeline
-    from .document_vl_embedding_pipeline import DocumentVLEmbeddingPipeline
-    from .video_captioning_pipeline import VideoCaptioningPipeline
-    from .video_question_answering_pipeline import VideoQuestionAnsweringPipeline
-    from .diffusers_wrapped import StableDiffusionPipeline, ChineseStableDiffusionPipeline
-    from .soonet_video_temporal_grounding_pipeline import SOONetVideoTemporalGroundingPipeline
-    from .text_to_video_synthesis_pipeline import TextToVideoSynthesisPipeline
-    from .multimodal_dialogue_pipeline import MultimodalDialoguePipeline
+    from .video_question_answering_pipeline import \
+        VideoQuestionAnsweringPipeline
+    from .visual_entailment_pipeline import VisualEntailmentPipeline
+    from .visual_grounding_pipeline import VisualGroundingPipeline
+    from .visual_question_answering_pipeline import \
+        VisualQuestionAnsweringPipeline
 else:
     _import_structure = {
         'image_captioning_pipeline': ['ImageCaptioningPipeline'],
         'visual_entailment_pipeline': ['VisualEntailmentPipeline'],
         'visual_grounding_pipeline': ['VisualGroundingPipeline'],
         'multi_modal_embedding_pipeline': ['MultiModalEmbeddingPipeline'],
+        'prost_text_video_retrieval_pipeline':
+        ['ProSTTextVideoRetrievalPipeline'],
         'text_to_image_synthesis_pipeline': ['TextToImageSynthesisPipeline'],
         'visual_question_answering_pipeline':
         ['VisualQuestionAnsweringPipeline'],

modelscope/pipelines/multi_modal/prost_text_video_retrieval_pipeline.py

@@ -0,0 +1,56 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from typing import Any, Dict
+
+from modelscope.metainfo import Pipelines
+from modelscope.pipelines.base import Input, Pipeline
+from modelscope.pipelines.builder import PIPELINES
+from modelscope.utils.constant import Tasks
+from modelscope.utils.device import device_placement
+from modelscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@PIPELINES.register_module(
+    Tasks.text_video_retrieval,
+    module_name=Pipelines.prost_text_video_retrieval)
+class ProSTTextVideoRetrievalPipeline(Pipeline):
+    '''
+    https://www.modelscope.cn/models/damo/multi_modal_clip_vtretrieval_prost/summary
+
+    from modelscope.pipelines import pipeline
+    from modelscope.utils.constant import Tasks
+    text_video_retrieval= pipeline(
+                Tasks.text_video_retrieval,
+                model='damo/multi_modal_clip_vtretrieval_prost')
+    video_path = 'https://modelscope.oss-cn-beijing.aliyuncs.com/test/videos/multi_modal_test_video_9770.mp4'
+    caption = 'a person is connecting something to system'
+    _input = {'video': video_path, 'text': caption}
+    result = text_video_retrieval(_input)
+    '''
+
+    def __init__(self, model: str, **kwargs):
+        """
+        use `model` to create a text_video_retrieval pipeline for prediction
+        Args:
+            model: model id on modelscope hub.
+        """
+        super().__init__(model=model)
+        self.model.eval()
+
+    def preprocess(self, input: Input) -> Dict[str, Any]:
+        return input
+
+    def _process_single(self, input: Input, *args, **kwargs) -> Dict[str, Any]:
+        with device_placement(self.framework, self.device_name):
+            out = self.forward(input)
+
+        self._check_output(out)
+        return out
+
+    def forward(self, input: Dict[str, Any]) -> Dict[str, Any]:
+        return self.model(input)
+
+    def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        return inputs

modelscope/utils/constant.py

@@ -243,6 +243,7 @@ class MultiModalTasks(object):
     visual_grounding = 'visual-grounding'
     text_to_image_synthesis = 'text-to-image-synthesis'
     multi_modal_embedding = 'multi-modal-embedding'
+    text_video_retrieval = 'text-video-retrieval'
     generative_multi_modal_embedding = 'generative-multi-modal-embedding'
     multi_modal_similarity = 'multi-modal-similarity'
     visual_question_answering = 'visual-question-answering'

tests/pipelines/test_prost_text_video_retrieval.py

@@ -0,0 +1,42 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import unittest
+
+# import modelscope
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+from modelscope.utils.logger import get_logger
+from modelscope.utils.test_utils import test_level
+
+logger = get_logger()
+
+
+class ProSTTextVideoRetrievalTest(unittest.TestCase):
+
+    def setUp(self) -> None:
+        self.task = Tasks.text_video_retrieval
+        self.model_id = 'damo/multi_modal_clip_vtretrieval_prost'
+
+    video_path = 'https://modelscope.oss-cn-beijing.aliyuncs.com/test/videos/multi_modal_test_video_9770.mp4'
+    caption = 'a person is connecting something to system'
+    # caption = 'a dog and a cat are friends'
+    _input = {'video': video_path, 'text': caption}
+
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_run(self):
+        pipeline_prost_text_video_retrieval = pipeline(
+            Tasks.text_video_retrieval, model=self.model_id)
+        output = pipeline_prost_text_video_retrieval(self._input)
+        logger.info('t2v sim: {}'.format(output['textvideo_sim']))
+        logger.info('phrase prototype: {}'.format(
+            output['phrase_prototype'].shape))
+        logger.info('object prototype: {}'.format(
+            output['object_prototype'].shape))
+        logger.info('sentence prototype: {}'.format(
+            output['sentence_prototype'].shape))
+        logger.info('event prototype: {}'.format(
+            output['event_prototype'].shape))
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/run_config.yaml

@@ -45,6 +45,7 @@ isolated:  # test cases that may require excessive anmount of GPU memory or run
   - test_table_recognition.py
   - test_conversational_text_to_sql.py
   - test_video_multi_modal_embedding.py
+  - test_prost_text_video_retrieval.py
   - test_image_skychange.py
   - test_video_stabilization.py
   - test_video_super_resolution.py