diff --git a/modelscope/fileio/format/jsonplus.py b/modelscope/fileio/format/jsonplus.py index 8608ce93..af59caeb 100644 --- a/modelscope/fileio/format/jsonplus.py +++ b/modelscope/fileio/format/jsonplus.py @@ -4,30 +4,29 @@ # TODO: handle environments without threads # (Python compiled without thread support) +import numpy as np import simplejson as json -from operator import attrgetter -from sortedcontainers import SortedList -from datetime import datetime, timedelta, date, time -from dateutil.parser import parse as parse_datetime -from functools import wraps, partial -from operator import methodcaller -from decimal import Decimal -from fractions import Fraction -from collections import namedtuple import threading import uuid -import numpy as np +from collections import namedtuple +from datetime import date, datetime, time, timedelta +from dateutil.parser import parse as parse_datetime +from decimal import Decimal +from fractions import Fraction +from functools import partial, wraps +from operator import attrgetter, methodcaller +from sortedcontainers import SortedList try: - from moneyed import Money, Currency + from moneyed import Currency, Money except ImportError: # defer failing to actual (de-)serialization pass -__all__ = ["loads", "dumps", "pretty", - "json_loads", "json_dumps", "json_prettydump", - "encoder", "decoder"] - +__all__ = [ + "loads", "dumps", "pretty", "json_loads", "json_dumps", "json_prettydump", + "encoder", "decoder" +] # Should we aim for the *exact* reproduction of Python types, # or for maximum *compatibility* when (de-)serializing? @@ -59,12 +58,15 @@ CODING_DEFAULT = EXACT _local = threading.local() + def prefer(coding): _local.coding = coding + def prefer_exact(): prefer(EXACT) + def prefer_compat(): prefer(COMPAT) @@ -103,15 +105,18 @@ def kwargified(constructor): >>> test({'b': 3}) 4 """ + @wraps(constructor) def kwargs_constructor(kwargs): return constructor(**kwargs) + return kwargs_constructor _PredicatedEncoder = namedtuple('_PredicatedEncoder', 'priority predicate encoder typename') + def encoder(classname, predicate=None, priority=None, exact=True): """A decorator for registering a new encoder for object type defined either by a `classname`, or detected via `predicate`. @@ -182,14 +187,18 @@ def _json_default_exact(obj): # first try predicate-based encoders for handler in _encode_handlers['exact']['predicate']: if handler.predicate(obj): - return {"__class__": handler.typename, - "__value__": handler.encoder(obj)} + return { + "__class__": handler.typename, + "__value__": handler.encoder(obj) + } # then classname-based classname = type(obj).__name__ if classname in _encode_handlers['exact']['classname']: - return {"__class__": classname, - "__value__": _encode_handlers['exact']['classname'][classname](obj)} + return { + "__class__": classname, + "__value__": _encode_handlers['exact']['classname'][classname](obj) + } raise TypeError(repr(obj) + " is not JSON serializable") @@ -217,8 +226,10 @@ def decoder(classname): def mytype_decoder(value): return mytype(value, reconstruct=True) """ + def _decorator(f): _decode_handlers.setdefault(classname, f) + return _decorator @@ -235,25 +246,25 @@ def _json_object_hook(dict): return dict - def _encoder_default_args(kw): """Shape default arguments for encoding functions.""" - + # manual override of the preferred coding with `exact=False` if kw.pop('exact', getattr(_local, 'coding', CODING_DEFAULT) == EXACT): # settings necessary for the "exact coding" kw.update({ 'default': _json_default_exact, - 'use_decimal': False, # don't encode `Decimal` as JSON's `Number` - 'tuple_as_array': False, # don't encode `tuple` as `Array` - 'namedtuple_as_object': False # don't call `_asdict` on `namedtuple` + 'use_decimal': False, # don't encode `Decimal` as JSON's `Number` + 'tuple_as_array': False, # don't encode `tuple` as `Array` + 'namedtuple_as_object': + False # don't call `_asdict` on `namedtuple` }) else: # settings for the "compatibility coding" kw.update({ 'default': _json_default_compat, - 'ignore_nan': True # be compliant with the ECMA-262 specification: - # serialize nan/inf as null + 'ignore_nan': True # be compliant with the ECMA-262 specification: + # serialize nan/inf as null }) # NOTE: if called from ``simplejson.dumps()`` with ``cls=JSONEncoder``, @@ -276,8 +287,8 @@ def _decoder_default_args(kw): kw.update({'object_hook': _json_object_hook}) - class JSONEncoder(json.JSONEncoder): + def __init__(self, **kw): """Constructor for simplejson.JSONEncoder, with defaults overriden for jsonplus. @@ -287,6 +298,7 @@ class JSONEncoder(json.JSONEncoder): class JSONDecoder(json.JSONDecoder): + def __init__(self, **kw): """Constructor for simplejson.JSONDecoder, with defaults overriden for jsonplus. @@ -295,7 +307,6 @@ class JSONDecoder(json.JSONDecoder): super(JSONDecoder, self).__init__(**kw) - def dumps(*pa, **kw): _encoder_default_args(kw) return json.dumps(*pa, **kw) @@ -306,14 +317,13 @@ def loads(*pa, **kw): return json.loads(*pa, **kw) -def pretty(x, sort_keys=True, indent=4*' ', separators=(',', ': '), **kw): +def pretty(x, sort_keys=True, indent=4 * ' ', separators=(',', ': '), **kw): kw.setdefault('sort_keys', sort_keys) kw.setdefault('indent', indent) kw.setdefault('separators', separators) return dumps(x, **kw) - json_dumps = dumps json_loads = loads json_prettydump = pretty @@ -330,21 +340,36 @@ def generic_to_item(value): _encode_handlers = { 'exact': { 'classname': { - 'datetime': methodcaller('isoformat'), - 'date': methodcaller('isoformat'), - 'time': methodcaller('isoformat'), - 'timedelta': partial(getattrs, attrs=['days', 'seconds', 'microseconds']), - 'tuple': list, - 'set': list, - 'ndarray': np_to_list, - 'float16': generic_to_item, - 'float32': generic_to_item, - 'frozenset': list, - 'complex': partial(getattrs, attrs=['real', 'imag']), - 'Decimal': str, - 'Fraction': partial(getattrs, attrs=['numerator', 'denominator']), - 'UUID': partial(getattrs, attrs=['hex']), - 'Money': partial(getattrs, attrs=['amount', 'currency']) + 'datetime': + methodcaller('isoformat'), + 'date': + methodcaller('isoformat'), + 'time': + methodcaller('isoformat'), + 'timedelta': + partial(getattrs, attrs=['days', 'seconds', 'microseconds']), + 'tuple': + list, + 'set': + list, + 'ndarray': + np_to_list, + 'float16': + generic_to_item, + 'float32': + generic_to_item, + 'frozenset': + list, + 'complex': + partial(getattrs, attrs=['real', 'imag']), + 'Decimal': + str, + 'Fraction': + partial(getattrs, attrs=['numerator', 'denominator']), + 'UUID': + partial(getattrs, attrs=['hex']), + 'Money': + partial(getattrs, attrs=['amount', 'currency']) }, 'predicate': SortedList(key=attrgetter('priority')) }, @@ -368,7 +393,6 @@ _encode_handlers = { } } - # all decode handlers are for EXACT decoding BY CLASSNAME _decode_handlers = { 'datetime': parse_datetime, @@ -388,11 +412,14 @@ _decode_handlers = { } -@encoder('namedtuple', lambda obj: isinstance(obj, tuple) and hasattr(obj, '_fields')) +@encoder('namedtuple', + lambda obj: isinstance(obj, tuple) and hasattr(obj, '_fields')) def _dump_namedtuple(obj): - return {"name": type(obj).__name__, - "fields": list(obj._fields), - "values": list(obj)} + return { + "name": type(obj).__name__, + "fields": list(obj._fields), + "values": list(obj) + } @decoder('namedtuple') @@ -404,7 +431,8 @@ def _load_namedtuple(val): @encoder('timedelta', exact=False) def _timedelta_total_seconds(td): # timedelta.total_seconds() is only available since python 2.7 - return (td.microseconds + (td.seconds + td.days * 24 * 3600.0) * 10**6) / 10**6 + return (td.microseconds + + (td.seconds + td.days * 24 * 3600.0) * 10**6) / 10**6 @encoder('Currency') @@ -412,7 +440,7 @@ def _dump_currency(obj): """Serialize standard (ISO-defined) currencies to currency code only, and non-standard (user-added) currencies in full. """ - from moneyed import get_currency, CurrencyDoesNotExist + from moneyed import CurrencyDoesNotExist, get_currency try: get_currency(obj.code) return obj.code diff --git a/modelscope/metainfo.py b/modelscope/metainfo.py index 99effb03..92702be8 100644 --- a/modelscope/metainfo.py +++ b/modelscope/metainfo.py @@ -213,6 +213,7 @@ class Models(object): video_synthesis = 'latent-text-to-video-synthesis' team = 'team-multi-modal-similarity' video_clip = 'video-clip-multi-modal-embedding' + prost = 'prost-clip-text-video-retrieval' mgeo = 'mgeo' vldoc = 'vldoc' hitea = 'hitea' @@ -528,6 +529,7 @@ class Pipelines(object): multi_modal_similarity = 'multi-modal-similarity' text_to_image_synthesis = 'text-to-image-synthesis' video_multi_modal_embedding = 'video-multi-modal-embedding' + prost_text_video_retrieval = 'prost-text-video-retrieval' image_text_retrieval = 'image-text-retrieval' ofa_ocr_recognition = 'ofa-ocr-recognition' ofa_asr = 'ofa-asr' @@ -720,6 +722,8 @@ DEFAULT_MODEL_FOR_PIPELINE = { Tasks.video_multi_modal_embedding: (Pipelines.video_multi_modal_embedding, 'damo/multi_modal_clip_vtretrival_msrvtt_53'), + Tasks.text_video_retrieval: (Pipelines.prost_text_video_retrieval, + 'damo/multi_modal_clip_vtretrieval_prost'), Tasks.image_color_enhancement: (Pipelines.image_color_enhance, 'damo/cv_csrnet_image-color-enhance-models'), diff --git a/modelscope/models/multi_modal/__init__.py b/modelscope/models/multi_modal/__init__.py index 9fa34baf..ab8ec91e 100644 --- a/modelscope/models/multi_modal/__init__.py +++ b/modelscope/models/multi_modal/__init__.py @@ -6,22 +6,23 @@ from modelscope.utils.import_utils import LazyImportModule if TYPE_CHECKING: from .clip import CLIPForMultiModalEmbedding - from .gemm import GEMMForMultiModalEmbedding - from .rleg import RLEGForMultiModalEmbedding - from .team import TEAMForMultiModalSimilarity + from .clip_interrogator import CLIP_Interrogator from .diffusion import DiffusionForTextToImageSynthesis + from .efficient_diffusion_tuning import EfficientStableDiffusion + from .gemm import GEMMForMultiModalEmbedding from .mmr import VideoCLIPForMultiModalEmbedding - from .mplug_for_all_tasks import MPlugForAllTasks, HiTeAForAllTasks + from .mplug_for_all_tasks import HiTeAForAllTasks, MPlugForAllTasks + from .mplug_owl import MplugOwlForConditionalGeneration + from .multi_stage_diffusion import \ + MultiStageDiffusionForTextToImageSynthesis from .ofa_for_all_tasks import OfaForAllTasks from .ofa_for_text_to_image_synthesis_model import \ OfaForTextToImageSynthesis - from .multi_stage_diffusion import \ - MultiStageDiffusionForTextToImageSynthesis - from .vldoc import VLDocForDocVLEmbedding + from .prost import ProSTForTVRetrieval + from .rleg import RLEGForMultiModalEmbedding + from .team import TEAMForMultiModalSimilarity from .video_synthesis import TextToVideoSynthesis - from .efficient_diffusion_tuning import EfficientStableDiffusion - from .mplug_owl import MplugOwlForConditionalGeneration - from .clip_interrogator import CLIP_Interrogator + from .vldoc import VLDocForDocVLEmbedding else: _import_structure = { @@ -31,6 +32,7 @@ else: 'rleg': ['RLEGForMultiModalEmbedding'], 'team': ['TEAMForMultiModalSimilarity'], 'mmr': ['VideoCLIPForMultiModalEmbedding'], + 'prost': ['ProSTForTVRetrieval'], 'mplug_for_all_tasks': ['MPlugForAllTasks', 'HiTeAForAllTasks'], 'ofa_for_all_tasks': ['OfaForAllTasks'], 'ofa_for_text_to_image_synthesis_model': diff --git a/modelscope/models/multi_modal/prost/__init__.py b/modelscope/models/multi_modal/prost/__init__.py new file mode 100644 index 00000000..db249cad --- /dev/null +++ b/modelscope/models/multi_modal/prost/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +from .models import ProSTForTVRetrieval diff --git a/modelscope/models/multi_modal/prost/dataloaders/__init__.py b/modelscope/models/multi_modal/prost/dataloaders/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modelscope/models/multi_modal/prost/dataloaders/rawvideo_util.py b/modelscope/models/multi_modal/prost/dataloaders/rawvideo_util.py new file mode 100644 index 00000000..c7ac3f94 --- /dev/null +++ b/modelscope/models/multi_modal/prost/dataloaders/rawvideo_util.py @@ -0,0 +1,117 @@ +# The implementation is adopted from Huaishao Luo, +# made pubicly available under the MIT License at https://github.com/ArrowLuo/CLIP4Clip + +import cv2 +import numpy as np +import torch as th +from PIL import Image +from torchvision.transforms import (CenterCrop, Compose, InterpolationMode, + Normalize, Resize, ToTensor) + +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +class RawVideoExtractorCV2(): + + def __init__( + self, + centercrop=False, + size=224, + frame_rate=-1, + ): + self.centercrop = centercrop + self.size = size + self.framerate = frame_rate + self.transform = self._transform(self.size) + + def _transform(self, n_px): + return Compose([ + Resize(n_px, interpolation=InterpolationMode.BICUBIC), + CenterCrop(n_px), + lambda image: image.convert('RGB'), + ToTensor(), + Normalize((0.48145466, 0.4578275, 0.40821073), + (0.26862954, 0.26130258, 0.27577711)), + ]) + + def video_to_tensor(self, + video_file, + preprocess, + sample_fp=0, + start_time=None, + end_time=None): + if start_time is not None or end_time is not None: + assert isinstance(start_time, int) and isinstance(end_time, int) \ + and start_time > -1 and end_time > start_time + assert sample_fp > -1 + + # Samples a frame sample_fp X frames. + cap = cv2.VideoCapture(video_file) + frameCount = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + fps = int(cap.get(cv2.CAP_PROP_FPS)) + + if fps == 0: + logger.info(f'{video_file} with fps 0!!!') + total_duration = (frameCount + fps - 1) // fps + start_sec, end_sec = 0, total_duration + + if start_time is not None: + start_sec, end_sec = start_time, end_time if end_time <= total_duration else total_duration + cap.set(cv2.CAP_PROP_POS_FRAMES, int(start_time * fps)) + + interval = 1 + if sample_fp > 0: + interval = fps // sample_fp + else: + sample_fp = fps + if interval == 0: + interval = 1 + + inds = [ind for ind in np.arange(0, fps, interval)] + assert len(inds) >= sample_fp + inds = inds[:sample_fp] + + ret = True + images = [] + + for sec in np.arange(start_sec, end_sec + 1): + if not ret: + break + sec_base = int(sec * fps) + for ind in inds: + cap.set(cv2.CAP_PROP_POS_FRAMES, sec_base + ind) + ret, frame = cap.read() + if not ret: + break + frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + images.append( + preprocess(Image.fromarray(frame_rgb).convert('RGB'))) + + cap.release() + + if len(images) > 0: + video_data = th.tensor(np.stack(images)) + else: + video_data = th.zeros(1) + return {'video': video_data} + + def get_video_data(self, video_path, start_time=None, end_time=None): + image_input = self.video_to_tensor( + video_path, + self.transform, + sample_fp=self.framerate, + start_time=start_time, + end_time=end_time) + return image_input + + def process_raw_data(self, raw_video_data): + tensor_size = raw_video_data.size() + tensor = raw_video_data.view(-1, 1, tensor_size[-3], tensor_size[-2], + tensor_size[-1]) + return tensor + + +# An ordinary video frame extractor based CV2 +RawVideoExtractor = RawVideoExtractorCV2 diff --git a/modelscope/models/multi_modal/prost/models/__init__.py b/modelscope/models/multi_modal/prost/models/__init__.py new file mode 100644 index 00000000..a35083a4 --- /dev/null +++ b/modelscope/models/multi_modal/prost/models/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +from .prost_model import ProSTForTVRetrieval diff --git a/modelscope/models/multi_modal/prost/models/modeling.py b/modelscope/models/multi_modal/prost/models/modeling.py new file mode 100644 index 00000000..b595f08b --- /dev/null +++ b/modelscope/models/multi_modal/prost/models/modeling.py @@ -0,0 +1,704 @@ +# Copyright 2021-2022 The Alibaba Fundamental Vision Team Authors. All rights reserved. + +import os +import platform +from collections import OrderedDict +from types import SimpleNamespace + +import torch +import torch.nn.functional as F +from torch import nn +from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence + +from modelscope.models.multi_modal.prost.models.module_clip import ( + _PT_NAME, CLIP, QuickGELU, convert_weights) +from modelscope.models.multi_modal.prost.models.module_cross import ( + CrossConfig, CrossModel) +from modelscope.models.multi_modal.prost.models.module_cross import \ + Transformer as TransformerClip +from modelscope.models.multi_modal.prost.models.until_module import ( + AllGather, CrossEn, Event_decoder, Frame_decoder, LayerNorm, + PreTrainedModel, make_patch_shift) +from modelscope.utils.logger import get_logger + +allgather = AllGather.apply + +logger = get_logger() +__all__ = ['CLIP4Clip'] + + +class MyObject: + + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + +class CLIP4ClipPreTrainedModel(PreTrainedModel, nn.Module): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + + def __init__(self, cross_config, *inputs, **kwargs): + super(CLIP4ClipPreTrainedModel, self).__init__(cross_config) + self.cross_config = cross_config + self.clip = None + self.cross = None + + @classmethod + def from_pretrained(cls, + cross_config, + state_dict=None, + cache_dir=None, + type_vocab_size=2, + *inputs, + **kwargs): + + task_config = None + if 'task_config' in kwargs.keys(): + task_config = kwargs['task_config'] + if not hasattr(task_config, 'local_rank'): + task_config['local_rank'] = 0 + elif task_config['local_rank'] == -1: + task_config['local_rank'] = 0 + + if state_dict is None: + state_dict = {} + # pretrained_clip_name = task_config['pretrained_clip_name'] + clip_state_dict = CLIP.get_config(model_dir=task_config['model_dir']) + for key, val in clip_state_dict.items(): + new_key = 'clip.' + key + if new_key not in state_dict: + state_dict[new_key] = val.clone() + + # cross_config, _ = CrossConfig.get_config( + # cross_model_name, + # cache_dir, + # type_vocab_size, + # state_dict=None, + # task_config=task_config) + cross_config = CrossConfig.from_dict(cross_config) + cross_config.type_vocab_size = type_vocab_size + task_config = MyObject(**kwargs['task_config']) + model = cls(cross_config, clip_state_dict, *inputs, task_config) + + # ===> Initialization trick [HARD CODE] + if model.linear_patch == '3d': + contain_conv2 = False + for key in state_dict.keys(): + if key.find('visual.conv2.weight') > -1: + contain_conv2 = True + break + if contain_conv2 is False and hasattr(model.clip.visual, 'conv2'): + cp_weight = state_dict['clip.visual.conv1.weight'].clone() + kernel_size = model.clip.visual.conv2.weight.size(2) + conv2_size = model.clip.visual.conv2.weight.size() + conv2_size = list(conv2_size) + + left_conv2_size = conv2_size.copy() + right_conv2_size = conv2_size.copy() + left_conv2_size[2] = (kernel_size - 1) // 2 + right_conv2_size[2] = kernel_size - 1 - left_conv2_size[2] + + left_zeros, right_zeros = None, None + if left_conv2_size[2] > 0: + left_zeros = torch.zeros( + *tuple(left_conv2_size), + dtype=cp_weight.dtype, + device=cp_weight.device) + if right_conv2_size[2] > 0: + right_zeros = torch.zeros( + *tuple(right_conv2_size), + dtype=cp_weight.dtype, + device=cp_weight.device) + + cat_list = [] + if left_zeros is not None: + cat_list.append(left_zeros) + cat_list.append(cp_weight.unsqueeze(2)) + if right_zeros is not None: + cat_list.append(right_zeros) + cp_weight = torch.cat(cat_list, dim=2) + + state_dict['clip.visual.conv2.weight'] = cp_weight + + # if model.sim_header == 'tightTransf': + # contain_cross = False + # for key in state_dict.keys(): + # if key.find('cross.transformer') > -1: + # contain_cross = True + # break + # if contain_cross is False: + # for key, val in clip_state_dict.items(): + # if key == 'positional_embedding': + # state_dict[ + # 'cross.embeddings.position_embeddings.weight'] = val.clone( + # ) + # continue + # if key.find('transformer.resblocks') == 0: + # num_layer = int(key.split('.')[2]) + + # # cut from beginning + # if num_layer < task_config.cross_num_hidden_layers: + # state_dict['cross.' + key] = val.clone() + # continue + + if model.sim_header == 'seqLSTM' or model.sim_header == 'seqTransf': + # This step is to detect whether in train mode or test mode + contain_frame_position = False + for key in state_dict.keys(): + if key.find('frame_position_embeddings') > -1: + contain_frame_position = True + break + + # train mode + if contain_frame_position is False: + for key, val in clip_state_dict.items(): + if key == 'positional_embedding': + state_dict[ + 'frame_position_embeddings.weight'] = val.clone() + # state_dict["text_prompt_encoder.pos_embedding"] = val[0:3].clone() + continue + if model.sim_header == 'seqTransf' and key.find( + 'transformer.resblocks') == 0: + num_layer = int(key.split('.')[2]) + # cut from beginning + if num_layer < task_config.cross_num_hidden_layers: + state_dict[key.replace( + 'transformer.', + 'transformerClip.')] = val.clone() + continue + + else: + for key, val in state_dict.items(): + # test mode + if key.find('clip.visual.transformer.resblocks') == 0: + num_layer = int(key.split('.')[4]) + # shift layers 10-11 + if num_layer >= 10 and num_layer < 12: + state_dict[key.replace('attn.net.', + 'attn.')] = val.clone() + # <=== End of initialization trick + + if state_dict is not None: + model = cls.init_preweight( + model, state_dict, task_config=task_config) + make_patch_shift(model, video_frame=task_config.max_frames, n_div=14) + return model + + +def show_log(task_config, info): + if task_config is None or task_config.local_rank == 0: + logger.warning(info) + + +def update_attr(target_name, + target_config, + target_attr_name, + source_config, + source_attr_name, + default_value=None): + if hasattr(source_config, source_attr_name): + if default_value is None or getattr(source_config, + source_attr_name) != default_value: + setattr(target_config, target_attr_name, + getattr(source_config, source_attr_name)) + # show_log( + # source_config, "Set {}.{}: {}.".format( + # target_name, target_attr_name, + # getattr(target_config, target_attr_name))) + return target_config + + +def check_attr(target_name, task_config): + return hasattr(task_config, + target_name) and task_config.__dict__[target_name] + + +class CLIP4Clip(CLIP4ClipPreTrainedModel): + + def __init__(self, cross_config, clip_state_dict, task_config): + super(CLIP4Clip, self).__init__(cross_config) + self.task_config = task_config + self.ignore_video_index = -1 + + assert self.task_config.max_words + self.task_config.max_frames <= cross_config.max_position_embeddings + + self._stage_one = True + self._stage_two = False + + # show_log(task_config, "Stage-One:{}, Stage-Two:{}".format(self._stage_one, self._stage_two)) + + self.loose_type = False + if self._stage_one and check_attr('loose_type', self.task_config): + self.loose_type = True + # show_log(task_config, "Test retrieval by loose type.") + + # CLIP Encoders: From OpenAI: CLIP [https://github.com/openai/CLIP] ===> + vit = 'visual.proj' in clip_state_dict + assert vit + if vit: + vision_width = clip_state_dict['visual.conv1.weight'].shape[0] + vision_layers = len([ + k for k in clip_state_dict.keys() if k.startswith('visual.') + and k.endswith('.attn.in_proj_weight') + ]) + vision_patch_size = clip_state_dict['visual.conv1.weight'].shape[ + -1] + grid_size = round( + (clip_state_dict['visual.positional_embedding'].shape[0] + - 1)**0.5) + image_resolution = vision_patch_size * grid_size + else: + counts: list = [ + len( + set( + k.split('.')[2] for k in clip_state_dict + if k.startswith(f'visual.layer{b}'))) + for b in [1, 2, 3, 4] + ] + vision_layers = tuple(counts) + vision_width = clip_state_dict[ + 'visual.layer1.0.conv1.weight'].shape[0] + output_width = round( + (clip_state_dict['visual.attnpool.positional_embedding']. + shape[0] - 1)**0.5) + vision_patch_size = None + assert output_width**2 + 1 == clip_state_dict[ + 'visual.attnpool.positional_embedding'].shape[0] + image_resolution = output_width * 32 + + embed_dim = clip_state_dict['text_projection'].shape[1] + context_length = clip_state_dict['positional_embedding'].shape[0] + vocab_size = clip_state_dict['token_embedding.weight'].shape[0] + transformer_width = clip_state_dict['ln_final.weight'].shape[0] + transformer_heads = transformer_width // 64 + transformer_layers = len( + set( + k.split('.')[2] for k in clip_state_dict + if k.startswith('transformer.resblocks'))) + + # show_log(task_config, "\t embed_dim: {}".format(embed_dim)) + # show_log(task_config, "\t image_resolution: {}".format(image_resolution)) + # show_log(task_config, "\t vision_layers: {}".format(vision_layers)) + # show_log(task_config, "\t vision_width: {}".format(vision_width)) + # show_log(task_config, "\t vision_patch_size: {}".format(vision_patch_size)) + # show_log(task_config, "\t context_length: {}".format(context_length)) + # show_log(task_config, "\t vocab_size: {}".format(vocab_size)) + # show_log(task_config, "\t transformer_width: {}".format(transformer_width)) + # show_log(task_config, "\t transformer_heads: {}".format(transformer_heads)) + # show_log(task_config, "\t transformer_layers: {}".format(transformer_layers)) + + self.linear_patch = '2d' + if hasattr(task_config, 'linear_patch'): + self.linear_patch = task_config.linear_patch + # show_log(task_config, "\t\t linear_patch: {}".format(self.linear_patch)) + + # use .float() to avoid overflow/underflow from fp16 weight. https://github.com/openai/CLIP/issues/40 + cut_top_layer = 0 + + self.clip = CLIP( + embed_dim, + image_resolution, + vision_layers - cut_top_layer, + vision_width, + vision_patch_size, + context_length, + vocab_size, + transformer_width, + transformer_heads, + transformer_layers - cut_top_layer, + linear_patch=self.linear_patch).float() + + for key in ['input_resolution', 'context_length', 'vocab_size']: + if key in clip_state_dict: + del clip_state_dict[key] + + convert_weights(self.clip) + # <=== End of CLIP Encoders + + self.sim_header = 'seqTransf' + if hasattr(task_config, 'sim_header'): + self.sim_header = task_config.sim_header + # show_log(task_config, "\t sim_header: {}".format(self.sim_header)) + if self.sim_header == 'tightTransf': + assert self.loose_type is False + + cross_config.max_position_embeddings = context_length + if self.loose_type is False: + # Cross Encoder ===> + cross_config = update_attr('cross_config', cross_config, + 'num_hidden_layers', self.task_config, + 'cross_num_hidden_layers') + self.cross = CrossModel(cross_config) + # <=== End of Cross Encoder + self.similarity_dense = nn.Linear(cross_config.hidden_size, 1) + + if self.sim_header == 'seqLSTM' or self.sim_header == 'seqTransf': + self.frame_position_embeddings = nn.Embedding( + cross_config.max_position_embeddings, cross_config.hidden_size) + # self.frame_position_embeddings = nn.Embedding(600, cross_config.hidden_size) + if self.sim_header == 'seqTransf': + self.transformerClip = TransformerClip( + width=transformer_width, + layers=self.task_config.cross_num_hidden_layers, + heads=transformer_heads, + ) + if self.sim_header == 'seqLSTM': + self.lstm_visual = nn.LSTM( + input_size=cross_config.hidden_size, + hidden_size=cross_config.hidden_size, + batch_first=True, + bidirectional=False, + num_layers=1) + + self.loss_fct = CrossEn() + self.apply(self.init_weights) + + self.set_dim = 512 + self.patch_num = self.task_config.max_patch + if hasattr(self.task_config, 'max_word_pro'): + self.word_pro_num = self.task_config.max_word_pro + else: + self.word_pro_num = self.task_config.max_phrase + + self.frame_num = self.task_config.max_frames + if hasattr(self.task_config, 'max_vfea'): + self.event_num = self.task_config.max_vfea + else: + self.event_num = self.task_config.max_event + + self.patch_prototype_weight = nn.Sequential( + nn.Linear(self.set_dim, self.set_dim), nn.ReLU(inplace=True), + nn.Linear(self.set_dim, self.patch_num - 1), nn.ReLU(inplace=True)) + + self.word_prototype_weight = nn.Sequential( + nn.Linear(self.set_dim, self.set_dim), nn.ReLU(inplace=True), + nn.Linear(self.set_dim, self.word_pro_num), nn.ReLU(inplace=True)) + + self.frame_decoder = Frame_decoder( + num_attris=self.frame_num, + layers=2, + heads=1, + dim_ftr=512, + pos_emb=False, + length=1, + dim_feedforward=512, + without_init=False) + self.event_decoder = Event_decoder( + num_attris=self.event_num, + layers=2, + heads=1, + dim_ftr=512, + pos_emb=False, + length=1, + dim_feedforward=512, + without_init=False) + # ------------------------------------------------------------------------------------------------------- + + def forward(self, + input_ids, + token_type_ids, + attention_mask, + video, + video_mask=None): + input_ids = input_ids.view(-1, input_ids.shape[-1]) + token_type_ids = token_type_ids.view(-1, token_type_ids.shape[-1]) + attention_mask = attention_mask.view(-1, attention_mask.shape[-1]) + video_mask = video_mask.view(-1, video_mask.shape[-1]) + + # T x 3 x H x W + video = torch.as_tensor(video).float() + bs, ts, channel, h, w = video.shape + video = video.view(bs * ts, channel, h, w) + video_frame = bs * ts + phr_feat, sen_feat, obj_feat, eve_feat = self.get_sequence_visual_output( + input_ids, + token_type_ids, + attention_mask, + video, + video_mask, + shaped=True, + video_frame=video_frame) + + if self.training: + sim_matrix1, sim_matrix2, sim_matrix3, sim_matrix4 = self.get_max_similarity_logits( + phr_feat, + sen_feat, + obj_feat, + eve_feat, + attention_mask, + video_mask, + shaped=True) + sim_loss = (self.loss_fct(sim_matrix1) + self.loss_fct(sim_matrix2) + + self.loss_fct(sim_matrix3) + + self.loss_fct(sim_matrix4)) / 4.0 + + loss = sim_loss + + return loss + else: + return None + + def get_max_similarity_logits(self, + word_feat, + text_feat, + patch_feat, + video_feat, + text_mask, + video_mask, + shaped=False): + if shaped is False: + text_mask = text_mask.view(-1, text_mask.shape[-1]) + video_mask = video_mask.view(-1, video_mask.shape[-1]) + + if self.training and torch.cuda.is_available(): # batch merge here + text_feat = allgather(text_feat, self.task_config) + video_feat = allgather(video_feat, self.task_config) + word_feat = allgather(word_feat, self.task_config) + patch_feat = allgather(patch_feat, self.task_config) + + video_mask = allgather(video_mask, self.task_config) + torch.distributed.barrier() # force sync + + # ESPM + text_feat = F.normalize(text_feat, p=2, dim=1) + video_feat = F.normalize(video_feat, p=2, dim=2) + retrieve_logits = torch.einsum('ad,bkd->abk', [text_feat, video_feat]) + retrieve_logits = retrieve_logits.max(2)[0] + + # OPPM + word_feat = F.normalize(word_feat, p=2, dim=2) + patch_feat = F.normalize(patch_feat, p=2, dim=3) + retrieve_logits_2 = torch.einsum('aid, bfjd->abfij', + [word_feat, patch_feat]) + + retrieve_logits_2 = retrieve_logits_2.max(3)[0] + retrieve_logits_2 = retrieve_logits_2.max(2)[0] + retrieve_logits_2 = retrieve_logits_2.sum(2) / self.patch_num + + if self.training: + logit_scale = self.clip.logit_scale.exp() + retrieve_logits = logit_scale * retrieve_logits + retrieve_logits_2 = logit_scale * retrieve_logits_2 + return retrieve_logits, retrieve_logits.t( + ), retrieve_logits_2, retrieve_logits_2.t() + + def get_sequence_output(self, + input_ids, + token_type_ids, + attention_mask, + shaped=False): + if shaped is False: + input_ids = input_ids.view(-1, input_ids.shape[-1]) + token_type_ids = token_type_ids.view(-1, token_type_ids.shape[-1]) + attention_mask = attention_mask.view(-1, attention_mask.shape[-1]) + bs_pair = input_ids.size(0) + sequence_hidden = self.clip.encode_text( + input_ids, return_hidden=True)[1].float() + text_feat = sequence_hidden.view(bs_pair, -1, sequence_hidden.size(-1)) + + word_weights = self.word_prototype_weight(text_feat) + text_word_proto = torch.einsum('bmd,bmn->bnd', text_feat, word_weights) + + cls_text_feat = text_feat.contiguous() + cls_text_feat = cls_text_feat[torch.arange(cls_text_feat.shape[0]), + torch.sum(attention_mask, dim=-1) - 1, :] + + return text_word_proto, cls_text_feat + + def get_visual_output(self, + video, + video_mask, + shaped=False, + video_frame=-1): + if shaped is False: + video_mask = video_mask.view(-1, video_mask.shape[-1]) + video = torch.as_tensor(video).float() + bs, ts, channel, h, w = video.shape + video = video.view(bs * ts, channel, h, w) + # video_frame = bs * ts + + bs_pair = video_mask.size(0) + + cls_video_feat, video_patch_feat = self.clip.encode_image_tokens( + video, return_hidden=True) + cls_video_feat = cls_video_feat.float() + video_patch_feat = video_patch_feat.float() + # frame_num = video_patch_feat.shape[0] + patch_dim = video_patch_feat.shape[2] + + patch_weights = self.patch_prototype_weight(video_patch_feat) + # cls_video_feat + video_patch_proto = torch.einsum('bmd,bmn->bnd', video_patch_feat, + patch_weights) + video_patch_proto = torch.cat( + (cls_video_feat.unsqueeze(1), video_patch_proto), 1) + video_patch_proto = video_patch_proto.reshape( + bs_pair, self.task_config.max_frames, self.patch_num, patch_dim) + + video_frame_proto = video_patch_proto.reshape( + bs_pair, self.patch_num * self.task_config.max_frames, patch_dim) + video_frame_proto = self.frame_decoder(video_frame_proto) + + video_frame_proto = 0.5 * video_frame_proto + 0.5 * cls_video_feat.reshape( + bs_pair, self.task_config.max_frames, patch_dim) + video_frame_proto = self.event_decoder(video_frame_proto) + video_frame_proto = 0.5 * video_frame_proto + 0.5 * cls_video_feat.reshape( + bs_pair, self.task_config.max_frames, patch_dim).mean(1).unsqueeze( + 1).repeat(1, video_frame_proto.shape[1], 1) + return video_patch_proto, video_frame_proto + + def get_sequence_visual_output(self, + input_ids, + token_type_ids, + attention_mask, + video, + video_mask, + shaped=False, + video_frame=-1): + if shaped is False: + input_ids = input_ids.view(-1, input_ids.shape[-1]) + token_type_ids = token_type_ids.view(-1, token_type_ids.shape[-1]) + attention_mask = attention_mask.view(-1, attention_mask.shape[-1]) + video_mask = video_mask.view(-1, video_mask.shape[-1]) + + video = torch.as_tensor(video).float() + + # import pdb;pdb.set_trace() + # b, pair, + bs, ts, channel, h, w = video.shape + video = video.view(bs * ts, channel, h, w) + video_frame = bs * ts + + word_feat, text_feat = self.get_sequence_output( + input_ids, token_type_ids, attention_mask, shaped=True) + + patch_feat, frame_feat = self.get_visual_output( + video, video_mask, shaped=True, video_frame=video_frame) + + return word_feat, text_feat, patch_feat, frame_feat + + def _get_cross_output(self, sequence_output, visual_output, attention_mask, + video_mask): + + concat_features = torch.cat((sequence_output, visual_output), + dim=1) # concatnate tokens and frames + concat_mask = torch.cat((attention_mask, video_mask), dim=1) + text_type_ = torch.zeros_like(attention_mask) + video_type_ = torch.ones_like(video_mask) + concat_type = torch.cat((text_type_, video_type_), dim=1) + + cross_layers, pooled_output = self.cross( + concat_features, + concat_type, + concat_mask, + output_all_encoded_layers=True) + cross_output = cross_layers[-1] + + return cross_output, pooled_output, concat_mask + + def _mean_pooling_for_similarity_sequence(self, sequence_output, + attention_mask): + attention_mask_un = attention_mask.to(dtype=torch.float).unsqueeze(-1) + attention_mask_un[:, 0, :] = 0. + sequence_output = sequence_output * attention_mask_un + text_out = torch.sum( + sequence_output, dim=1) / torch.sum( + attention_mask_un, dim=1, dtype=torch.float) + return text_out + + def _mean_pooling_for_similarity_visual( + self, + visual_output, + video_mask, + ): + video_mask_un = video_mask.to(dtype=torch.float).unsqueeze(-1) + visual_output = visual_output * video_mask_un + video_mask_un_sum = torch.sum(video_mask_un, dim=1, dtype=torch.float) + video_mask_un_sum[video_mask_un_sum == 0.] = 1. + video_out = torch.sum(visual_output, dim=1) / video_mask_un_sum + return video_out + + def _mean_pooling_for_similarity( + self, + sequence_output, + visual_output, + attention_mask, + video_mask, + ): + text_out = self._mean_pooling_for_similarity_sequence( + sequence_output, attention_mask) + video_out = self._mean_pooling_for_similarity_visual( + visual_output, video_mask) + + return text_out, video_out + + def get_global_similarity(self, sequence_output, visual_output, + attention_mask, video_mask): + visual_output = visual_output / visual_output.norm( + dim=-1, keepdim=True) + visual_output = self._mean_pooling_for_similarity_visual( + visual_output, video_mask) + visual_output = visual_output / visual_output.norm( + dim=-1, keepdim=True) + + sequence_output = sequence_output.squeeze(1) + sequence_output = sequence_output / sequence_output.norm( + dim=-1, keepdim=True) + + logit_scale = self.clip.logit_scale.exp() + # retrieve_logits = logit_scale * torch.matmul(sequence_output, visual_output.t()) + sim_matrix_global = logit_scale * torch.matmul(sequence_output, + visual_output.t()) + return sim_matrix_global + + def _cross_similarity(self, sequence_output, visual_output, attention_mask, + video_mask): + sequence_output, visual_output = sequence_output.contiguous( + ), visual_output.contiguous() + + b_text, s_text, h_text = sequence_output.size() + b_visual, s_visual, h_visual = visual_output.size() + + retrieve_logits_list = [] + + step_size = b_text # set smaller to reduce memory cost + split_size = [step_size] * (b_text // step_size) + release_size = b_text - sum(split_size) + if release_size > 0: + split_size += [release_size] + + # due to clip text branch retrun the last hidden + attention_mask = torch.ones(sequence_output.size(0), 1)\ + .to(device=attention_mask.device, dtype=attention_mask.dtype) + + sequence_output_splits = torch.split( + sequence_output, split_size, dim=0) + attention_mask_splits = torch.split(attention_mask, split_size, dim=0) + for i in range(len(split_size)): + sequence_output_row = sequence_output_splits[i] + attention_mask_row = attention_mask_splits[i] + sequence_output_l = sequence_output_row.unsqueeze(1).repeat( + 1, b_visual, 1, 1) + sequence_output_l = sequence_output_l.view(-1, s_text, h_text) + attention_mask_l = attention_mask_row.unsqueeze(1).repeat( + 1, b_visual, 1) + attention_mask_l = attention_mask_l.view(-1, s_text) + + step_truth = sequence_output_row.size(0) + visual_output_r = visual_output.unsqueeze(0).repeat( + step_truth, 1, 1, 1) + visual_output_r = visual_output_r.view(-1, s_visual, h_visual) + video_mask_r = video_mask.unsqueeze(0).repeat(step_truth, 1, 1) + video_mask_r = video_mask_r.view(-1, s_visual) + + cross_output, pooled_output, concat_mask = \ + self._get_cross_output(sequence_output_l, visual_output_r, attention_mask_l, video_mask_r) + retrieve_logits_row = self.similarity_dense(pooled_output).squeeze( + -1).view(step_truth, b_visual) + + retrieve_logits_list.append(retrieve_logits_row) + + retrieve_logits = torch.cat(retrieve_logits_list, dim=0) + return retrieve_logits diff --git a/modelscope/models/multi_modal/prost/models/module_clip.py b/modelscope/models/multi_modal/prost/models/module_clip.py new file mode 100644 index 00000000..c5aaa1e5 --- /dev/null +++ b/modelscope/models/multi_modal/prost/models/module_clip.py @@ -0,0 +1,538 @@ +# The implementation is adopated from the CLIP4Clip implementation, +# made pubicly available under Apache License, Version 2.0 at https://github.com/ArrowLuo/CLIP4Clip + +import hashlib +import os +import urllib +import warnings +from collections import OrderedDict +from typing import Tuple, Union + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +from torch import nn +from tqdm import tqdm + +_MODELS = {} +_PT_NAME = {'ViT-B/16': 'ViT-B-16.pt'} + + +def available_models(): + """Returns the names of available CLIP models""" + return list(_MODELS.keys()) + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1): + super(Bottleneck, self).__init__() + + # all conv layers have stride 1. an avgpool is performed after the second convolution when stride > 1 + self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + + self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + + self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity() + + self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + + self.relu = nn.ReLU(inplace=True) + self.downsample = None + self.stride = stride + + if stride > 1 or inplanes != planes * Bottleneck.expansion: + # downsampling layer is prepended with an avgpool, and the subsequent convolution has stride 1 + self.downsample = nn.Sequential( + OrderedDict([('-1', nn.AvgPool2d(stride)), + ('0', + nn.Conv2d( + inplanes, + planes * self.expansion, + 1, + stride=1, + bias=False)), + ('1', nn.BatchNorm2d(planes * self.expansion))])) + + def forward(self, x: torch.Tensor): + identity = x + + out = self.relu(self.bn1(self.conv1(x))) + out = self.relu(self.bn2(self.conv2(out))) + out = self.avgpool(out) + out = self.bn3(self.conv3(out)) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + return out + + +class AttentionPool2d(nn.Module): + + def __init__(self, + spacial_dim: int, + embed_dim: int, + num_heads: int, + output_dim: int = None): + super(AttentionPool2d, self).__init__() + self.positional_embedding = nn.Parameter( + torch.randn(spacial_dim**2 + 1, embed_dim) / embed_dim**0.5) + self.k_proj = nn.Linear(embed_dim, embed_dim) + self.q_proj = nn.Linear(embed_dim, embed_dim) + self.v_proj = nn.Linear(embed_dim, embed_dim) + self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim) + self.num_heads = num_heads + + def forward(self, x): + x = x.reshape(x.shape[0], x.shape[1], + x.shape[2] * x.shape[3]).permute(2, 0, + 1) # NCHW -> (HW)NC + x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (HW+1)NC + x = x + self.positional_embedding[:, None, :].to(x.dtype) # (HW+1)NC + x, _ = F.multi_head_attention_forward( + query=x, + key=x, + value=x, + embed_dim_to_check=x.shape[-1], + num_heads=self.num_heads, + q_proj_weight=self.q_proj.weight, + k_proj_weight=self.k_proj.weight, + v_proj_weight=self.v_proj.weight, + in_proj_weight=None, + in_proj_bias=torch.cat( + [self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]), + bias_k=None, + bias_v=None, + add_zero_attn=False, + dropout_p=0, + out_proj_weight=self.c_proj.weight, + out_proj_bias=self.c_proj.bias, + use_separate_proj_weight=True, + training=self.training, + need_weights=False) + + return x[0] + + +class ModifiedResNet(nn.Module): + """ + A ResNet class that is similar to torchvision's but contains the following changes: + - There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool. + - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1 + - The final pooling layer is a QKV attention instead of an average pool + """ + + def __init__(self, + layers, + output_dim, + heads, + input_resolution=224, + width=64): + super(ModifiedResNet, self).__init__() + self.output_dim = output_dim + self.input_resolution = input_resolution + + # the 3-layer stem + self.conv1 = nn.Conv2d( + 3, width // 2, kernel_size=3, stride=2, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(width // 2) + self.conv2 = nn.Conv2d( + width // 2, width // 2, kernel_size=3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(width // 2) + self.conv3 = nn.Conv2d( + width // 2, width, kernel_size=3, padding=1, bias=False) + self.bn3 = nn.BatchNorm2d(width) + self.avgpool = nn.AvgPool2d(2) + self.relu = nn.ReLU(inplace=True) + + # residual layers + self._inplanes = width # this is a *mutable* variable used during construction + self.layer1 = self._make_layer(width, layers[0]) + self.layer2 = self._make_layer(width * 2, layers[1], stride=2) + self.layer3 = self._make_layer(width * 4, layers[2], stride=2) + self.layer4 = self._make_layer(width * 8, layers[3], stride=2) + + embed_dim = width * 32 # the ResNet feature dimension + self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim, + heads, output_dim) + + def _make_layer(self, planes, blocks, stride=1): + layers = [Bottleneck(self._inplanes, planes, stride)] + + self._inplanes = planes * Bottleneck.expansion + for _ in range(1, blocks): + layers.append(Bottleneck(self._inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x): + + def stem(x): + for conv, bn in [(self.conv1, self.bn1), (self.conv2, self.bn2), + (self.conv3, self.bn3)]: + x = self.relu(bn(conv(x))) + x = self.avgpool(x) + return x + + x = x.type(self.conv1.weight.dtype) + x = stem(x) + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + x = self.attnpool(x) + + return x + + +class LayerNorm(nn.LayerNorm): + """Subclass torch's LayerNorm to handle fp16.""" + + def forward(self, x: torch.Tensor): + orig_type = x.dtype + ret = super().forward(x.type(torch.float32)) + return ret.type(orig_type) + + +class QuickGELU(nn.Module): + + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +class ResidualAttentionBlock(nn.Module): + + def __init__(self, d_model: int, n_head: int, attn_mask=None): + super(ResidualAttentionBlock, self).__init__() + + self.attn = nn.MultiheadAttention(d_model, n_head) + self.ln_1 = LayerNorm(d_model) + self.mlp = nn.Sequential( + OrderedDict([('c_fc', nn.Linear(d_model, d_model * 4)), + ('gelu', QuickGELU()), + ('c_proj', nn.Linear(d_model * 4, d_model))])) + self.ln_2 = LayerNorm(d_model) + self.attn_mask = attn_mask + + def attention(self, x: torch.Tensor): + attn_mask_ = self.attn_mask + if self.attn_mask is not None and hasattr(self.attn_mask, '__call__'): + attn_mask_ = self.attn_mask(x.size(0)) # LND + + attn_mask_ = attn_mask_.to( + dtype=x.dtype, device=x.device) if attn_mask_ is not None else None + return self.attn(x, x, x, need_weights=False, attn_mask=attn_mask_)[0] + + def forward(self, x): + x = x + self.attention(self.ln_1(x)) + x = x + self.mlp(self.ln_2(x)) + return x + + +class Transformer(nn.Module): + + def __init__(self, + width: int, + layers: int, + heads: int, + attn_mask=None, + use_gc=0): + super(Transformer, self).__init__() + self.width = width + self.layers = layers + self.resblocks = nn.Sequential(*[ + ResidualAttentionBlock(width, heads, attn_mask) + for _ in range(layers) + ]) + + self.use_gc = use_gc + + def forward(self, x: torch.Tensor): + if self.use_gc > 0: + for blk in self.resblocks: + x = checkpoint.checkpoint(blk, x) + return x + else: + return self.resblocks(x) + + +class VisualTransformer(nn.Module): + + def __init__(self, + input_resolution: int, + patch_size: int, + width: int, + layers: int, + heads: int, + output_dim: int, + linear_patch: str = '2d', + use_gc: int = 0): + super(VisualTransformer, self).__init__() + self.input_resolution = input_resolution + self.output_dim = output_dim + + self.conv1 = nn.Conv2d( + in_channels=3, + out_channels=width, + kernel_size=patch_size, + stride=patch_size, + bias=False) + + scale = width**-0.5 + self.class_embedding = nn.Parameter(scale * torch.randn(width)) + self.positional_embedding = nn.Parameter(scale * torch.randn( + (input_resolution // patch_size)**2 + 1, width)) + self.ln_pre = LayerNorm(width) + + self.transformer = Transformer(width, layers, heads, use_gc=use_gc) + + self.ln_post = LayerNorm(width) + self.proj = nn.Parameter(scale * torch.randn(width, output_dim)) + + # For 3D + assert linear_patch in ['2d', '3d'] + self.linear_patch = linear_patch + if self.linear_patch == '3d': + self.conv2 = nn.Conv3d( + in_channels=3, + out_channels=width, + kernel_size=(3, patch_size, patch_size), + stride=(1, patch_size, patch_size), + padding=(1, 0, 0), + bias=False) + + def forward(self, x: torch.Tensor, video_frame=-1): + + if self.linear_patch == '3d': + assert video_frame != -1 + x_3d = x.reshape(-1, video_frame, x.shape[-3], x.shape[-2], + x.shape[-1]) + x_3d = x_3d.permute(0, 2, 1, 3, 4) + x_3d = self.conv2(x_3d) # shape = [*, width, frame, grid, grid] + x_3d = x_3d.permute(0, 2, 1, 3, + 4) # shape = [*, frame, width, grid, grid] + x = x_3d.reshape( + -1, x_3d.shape[-3], x_3d.shape[-2], + x_3d.shape[-1]).contiguous() # shape = [*, width, grid, grid] + else: + x = self.conv1(x) # shape = [*, width, grid, grid] + + x = x.reshape(x.shape[0], x.shape[1], + -1) # shape = [*, width, grid ** 2] + x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width] + + _x = self.class_embedding.to(x.dtype) + torch.zeros( + x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device) + x = torch.cat([_x, x], dim=1) + x = x + self.positional_embedding.to(x.dtype) + x = self.ln_pre(x) + + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + + return x + + +class CLIP(nn.Module): + + def __init__( + self, + embed_dim: int, + # vision + image_resolution: int, + vision_layers: Union[Tuple[int, int, int, int], int], + vision_width: int, + vision_patch_size: int, + # text + context_length: int, + vocab_size: int, + transformer_width: int, + transformer_heads: int, + transformer_layers: int, + # vision linear of patch + linear_patch: str = '2d', + use_gc: int = 0): + super(CLIP, self).__init__() + + self.context_length = context_length + + if isinstance(vision_layers, (tuple, list)): + vision_heads = vision_width * 32 // 64 + self.visual = ModifiedResNet( + layers=vision_layers, + output_dim=embed_dim, + heads=vision_heads, + input_resolution=image_resolution, + width=vision_width) + else: + vision_heads = vision_width // 64 + self.visual = VisualTransformer( + input_resolution=image_resolution, + patch_size=vision_patch_size, + width=vision_width, + layers=vision_layers, + heads=vision_heads, + output_dim=embed_dim, + linear_patch=linear_patch, + use_gc=use_gc) + + self.transformer = Transformer( + width=transformer_width, + layers=transformer_layers, + heads=transformer_heads, + attn_mask=self.build_attention_mask) + + self.vocab_size = vocab_size + self.token_embedding = nn.Embedding(vocab_size, transformer_width) + self.positional_embedding = nn.Parameter( + torch.empty(self.context_length, transformer_width)) + self.ln_final = LayerNorm(transformer_width) + + self.text_projection = nn.Parameter( + torch.empty(transformer_width, embed_dim)) + self.logit_scale = nn.Parameter(torch.ones([])) + + self.initialize_parameters() + + def initialize_parameters(self): + nn.init.normal_(self.token_embedding.weight, std=0.02) + nn.init.normal_(self.positional_embedding, std=0.01) + + if isinstance(self.visual, ModifiedResNet): + if self.visual.attnpool is not None: + std = self.visual.attnpool.c_proj.in_features**-0.5 + nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std) + + for resnet_block in [ + self.visual.layer1, self.visual.layer2, self.visual.layer3, + self.visual.layer4 + ]: + for name, param in resnet_block.named_parameters(): + if name.endswith('bn3.weight'): + nn.init.zeros_(param) + + proj_std = (self.transformer.width**-0.5) * ( + (2 * self.transformer.layers)**-0.5) + attn_std = self.transformer.width**-0.5 + fc_std = (2 * self.transformer.width)**-0.5 + for block in self.transformer.resblocks: + nn.init.normal_(block.attn.in_proj_weight, std=attn_std) + nn.init.normal_(block.attn.out_proj.weight, std=proj_std) + nn.init.normal_(block.mlp.c_fc.weight, std=fc_std) + nn.init.normal_(block.mlp.c_proj.weight, std=proj_std) + + if self.text_projection is not None: + nn.init.normal_( + self.text_projection, std=self.transformer.width**-0.5) + + def build_attention_mask(self, context_length): + # lazily create causal attention mask, with full attention between the vision tokens + # pytorch uses additive attention mask; fill with -inf + mask = torch.zeros(context_length, context_length) + mask.fill_(float('-inf')) + mask.triu_(1) # zero out the lower diagonal + return mask + + @staticmethod + def get_config(model_dir): + model_path = '{}/ViT-B-16.pt'.format(model_dir) + try: + # loading JIT archive + model = torch.jit.load(model_path, map_location='cpu').eval() + state_dict = model.state_dict() + except RuntimeError: + state_dict = torch.load(model_path, map_location='cpu') + return state_dict + + @property + def dtype(self): + return self.visual.conv1.weight.dtype + + def encode_image_tokens(self, image, return_hidden=False): + hidden = self.visual(image.type(self.dtype)) + hidden = self.visual.ln_post(hidden) @ self.visual.proj + + x = hidden[:, 0, :] + + if return_hidden: + return x, hidden + + return x + + def encode_text(self, text, return_hidden=False, prompt=None): + x = self.token_embedding(text).type( + self.dtype) # [batch_size, n_ctx, d_model] + if prompt: + x = prompt(x) + + pos_emd = self.positional_embedding[:x.size(1), :].type(self.dtype) + x = x + pos_emd + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + + hidden = self.ln_final(x).type(self.dtype) @ self.text_projection + + # take features from the eot embedding (eot_token is the highest number in each sequence) + x = hidden[torch.arange(hidden.shape[0]), text.argmax(dim=-1)] + + if return_hidden: + return x, hidden + + return x + + def forward(self, image, text): + image_features = self.encode_image(image) + text_features = self.encode_text(text) + + # normalized features + image_features = image_features / image_features.norm( + dim=-1, keepdim=True) + text_features = text_features / text_features.norm( + dim=-1, keepdim=True) + + # cosine similarity as logits + logit_scale = self.logit_scale.exp() + logits_per_image = logit_scale * image_features @ text_features.t() + logits_per_text = logit_scale * text_features @ image_features.t() + + return logits_per_image, logits_per_text + + +def convert_weights(model: nn.Module): + """Convert applicable model parameters to fp16""" + + def _convert_weights_to_fp16(lay): + # l = lay + if isinstance(lay, (nn.Conv1d, nn.Conv2d, nn.Conv3d, nn.Linear)): + lay.weight.data = lay.weight.data.half() + if lay.bias is not None: + lay.bias.data = lay.bias.data.half() + + if isinstance(lay, nn.MultiheadAttention): + for attr in [ + *[f'{s}_proj_weight' for s in ['in', 'q', 'k', 'v']], + 'in_proj_bias', 'bias_k', 'bias_v' + ]: + tensor = getattr(lay, attr) + if tensor is not None: + tensor.data = tensor.data.half() + + for name in ['text_projection', 'proj']: + if hasattr(lay, name): + attr = getattr(lay, name) + if attr is not None: + attr.data = attr.data.half() + + model.apply(_convert_weights_to_fp16) diff --git a/modelscope/models/multi_modal/prost/models/module_cross.py b/modelscope/models/multi_modal/prost/models/module_cross.py new file mode 100644 index 00000000..fae8e904 --- /dev/null +++ b/modelscope/models/multi_modal/prost/models/module_cross.py @@ -0,0 +1,249 @@ +from __future__ import absolute_import, division, print_function +import copy +import logging +import math +import os +import shutil +import tarfile +import tempfile +from collections import OrderedDict + +import json +import torch +import torch.nn.functional as F +from torch import nn + +from .until_config import PreCrossConfig +from .until_module import ACT2FN, LayerNorm, PreTrainedModel + + +# PRETRAINED_MODEL_ARCHIVE_MAP = {} +# CONFIG_NAME = 'cross_config.json' +# WEIGHTS_NAME = 'cross_pytorch_model.bin' +class CrossConfig(PreCrossConfig): + """Configuration class to store the configuration of a `CrossModel`. + """ + + # pretrained_model_archive_map = PRETRAINED_MODEL_ARCHIVE_MAP + # config_name = CONFIG_NAME + # weights_name = WEIGHTS_NAME + def __init__(self, + vocab_size_or_config_json_file, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act='gelu', + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02): + """Constructs CrossConfig. + + Args: + vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `CrossModel`. + hidden_size: Size of the encoder layers and the pooler layer. + num_hidden_layers: Number of hidden layers in the Transformer encoder. + num_attention_heads: Number of attention heads for each attention layer in + the Transformer encoder. + intermediate_size: The size of the "intermediate" (i.e., feed-forward) + layer in the Transformer encoder. + hidden_act: The non-linear activation function (function or string) in the + encoder and pooler. If string, "gelu", "relu" and "swish" are supported. + hidden_dropout_prob: The dropout probabilitiy for all fully connected + layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob: The dropout ratio for the attention + probabilities. + max_position_embeddings: The maximum sequence length that this model might + ever be used with. Typically set this to something large just in case + (e.g., 512 or 1024 or 2048). + type_vocab_size: The vocabulary size of the `token_type_ids` passed into + `CrossModel`. + initializer_range: The sttdev of the truncated_normal_initializer for + initializing all weight matrices. + """ + if isinstance(vocab_size_or_config_json_file, str): + with open( + vocab_size_or_config_json_file, 'r', + encoding='utf-8') as reader: + json_config = json.loads(reader.read()) + for key, value in json_config.items(): + self.__dict__[key] = value + elif isinstance(vocab_size_or_config_json_file, int): + self.vocab_size = vocab_size_or_config_json_file + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + else: + raise ValueError( + 'First argument must be either a vocabulary size (int)' + 'or the path to a pretrained model config file (str)') + + +class QuickGELU(nn.Module): + + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +class ResidualAttentionBlock(nn.Module): + + def __init__(self, d_model: int, n_head: int): + super().__init__() + + self.attn = nn.MultiheadAttention(d_model, n_head) + self.ln_1 = LayerNorm(d_model) + self.mlp = nn.Sequential( + OrderedDict([('c_fc', nn.Linear(d_model, d_model * 4)), + ('gelu', QuickGELU()), + ('c_proj', nn.Linear(d_model * 4, d_model))])) + self.ln_2 = LayerNorm(d_model) + self.n_head = n_head + + def attention(self, x: torch.Tensor, attn_mask: torch.Tensor): + attn_mask_ = attn_mask.repeat_interleave(self.n_head, dim=0) + return self.attn(x, x, x, need_weights=False, attn_mask=attn_mask_)[0] + + def forward(self, para_tuple: tuple): + # x: torch.Tensor, attn_mask: torch.Tensor + # print(para_tuple) + x, attn_mask = para_tuple + x = x + self.attention(self.ln_1(x), attn_mask) + x = x + self.mlp(self.ln_2(x)) + return (x, attn_mask) + + +class Transformer(nn.Module): + + def __init__(self, width: int, layers: int, heads: int): + super().__init__() + self.width = width + self.layers = layers + self.resblocks = nn.Sequential( + *[ResidualAttentionBlock(width, heads) for _ in range(layers)]) + + def forward(self, x: torch.Tensor, attn_mask: torch.Tensor): + return self.resblocks((x, attn_mask))[0] + + +class CrossEmbeddings(nn.Module): + """Construct the embeddings from word, position and token_type embeddings. + """ + + def __init__(self, config): + super(CrossEmbeddings, self).__init__() + + self.position_embeddings = nn.Embedding(config.max_position_embeddings, + config.hidden_size) + # self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size) + # self.LayerNorm = LayerNorm(config.hidden_size, eps=1e-12) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, concat_embeddings, concat_type=None): + + _, seq_length = concat_embeddings.size(0), concat_embeddings.size(1) + # if concat_type is None: + # concat_type = torch.zeros(batch_size, concat_type).to(concat_embeddings.device) + + position_ids = torch.arange( + seq_length, dtype=torch.long, device=concat_embeddings.device) + position_ids = position_ids.unsqueeze(0).expand( + concat_embeddings.size(0), -1) + + # token_type_embeddings = self.token_type_embeddings(concat_type) + position_embeddings = self.position_embeddings(position_ids) + + embeddings = concat_embeddings + position_embeddings # + token_type_embeddings + # embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + + +class CrossPooler(nn.Module): + + def __init__(self, config): + super(CrossPooler, self).__init__() + self.ln_pool = LayerNorm(config.hidden_size) + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = QuickGELU() + + def forward(self, hidden_states, hidden_mask): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + hidden_states = self.ln_pool(hidden_states) + pooled_output = hidden_states[:, 0] + pooled_output = self.dense(pooled_output) + pooled_output = self.activation(pooled_output) + return pooled_output + + +class CrossModel(PreTrainedModel): + + def initialize_parameters(self): + proj_std = (self.transformer.width**-0.5) * ( + (2 * self.transformer.layers)**-0.5) + attn_std = self.transformer.width**-0.5 + fc_std = (2 * self.transformer.width)**-0.5 + for block in self.transformer.resblocks: + nn.init.normal_(block.attn.in_proj_weight, std=attn_std) + nn.init.normal_(block.attn.out_proj.weight, std=proj_std) + nn.init.normal_(block.mlp.c_fc.weight, std=fc_std) + nn.init.normal_(block.mlp.c_proj.weight, std=proj_std) + + def __init__(self, config): + super(CrossModel, self).__init__(config) + + self.embeddings = CrossEmbeddings(config) + + transformer_width = config.hidden_size + transformer_layers = config.num_hidden_layers + transformer_heads = config.num_attention_heads + self.transformer = Transformer( + width=transformer_width, + layers=transformer_layers, + heads=transformer_heads, + ) + self.pooler = CrossPooler(config) + self.apply(self.init_weights) + + def build_attention_mask(self, attention_mask): + extended_attention_mask = attention_mask.unsqueeze(1) + extended_attention_mask = extended_attention_mask.to( + dtype=self.dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -1000000.0 + extended_attention_mask = extended_attention_mask.expand( + -1, attention_mask.size(1), -1) + return extended_attention_mask + + def forward(self, + concat_input, + concat_type=None, + attention_mask=None, + output_all_encoded_layers=True): + + if attention_mask is None: + attention_mask = torch.ones( + concat_input.size(0), concat_input.size(1)) + if concat_type is None: + concat_type = torch.zeros_like(attention_mask) + + extended_attention_mask = self.build_attention_mask(attention_mask) + + embedding_output = self.embeddings(concat_input, concat_type) + embedding_output = embedding_output.permute(1, 0, 2) # NLD -> LND + embedding_output = self.transformer(embedding_output, + extended_attention_mask) + embedding_output = embedding_output.permute(1, 0, 2) # LND -> NLD + + pooled_output = self.pooler( + embedding_output, hidden_mask=attention_mask) + + return embedding_output, pooled_output diff --git a/modelscope/models/multi_modal/prost/models/prost_model.py b/modelscope/models/multi_modal/prost/models/prost_model.py new file mode 100644 index 00000000..022903cb --- /dev/null +++ b/modelscope/models/multi_modal/prost/models/prost_model.py @@ -0,0 +1,267 @@ +# The implementation is adopted from the CLIP4Clip implementation, +# made pubicly available under Apache License, Version 2.0 at https://github.com/ArrowLuo/CLIP4Clip + +import os +import random +import uuid +from os.path import exists +from tempfile import TemporaryDirectory +from typing import Any, Dict +from urllib.parse import urlparse + +import json +import numpy as np +import torch +from decord import VideoReader, cpu +from PIL import Image + +from modelscope.hub.file_download import http_get_file +from modelscope.metainfo import Models +from modelscope.models import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.multi_modal.prost.models.modeling import CLIP4Clip +from modelscope.models.multi_modal.prost.models.tokenization_clip import \ + SimpleTokenizer as ClipTokenizer +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from ..dataloaders.rawvideo_util import RawVideoExtractor + +logger = get_logger() + + +@MODELS.register_module(Tasks.text_video_retrieval, module_name=Models.prost) +class ProSTForTVRetrieval(TorchModel): + + def __init__(self, model_dir, **kwargs): + super().__init__(model_dir=model_dir, **kwargs) + # model config parameters + with open( + f'{model_dir}/{ModelFile.CONFIGURATION}', 'r', + encoding='utf-8') as json_file: + all_model_config = json.load(json_file) + model_config = all_model_config['paras'] + + cross_model_config = all_model_config['crossbase'] + # print(all_model_config) + # print(cross_model_config) + model_config['model_dir'] = model_dir + self.SPECIAL_TOKEN = { + 'CLS_TOKEN': '<|startoftext|>', + 'SEP_TOKEN': '<|endoftext|>', + 'MASK_TOKEN': '[MASK]', + 'UNK_TOKEN': '[UNK]', + 'PAD_TOKEN': '[PAD]' + } + self.max_words = model_config['max_words'] + self.max_frames = model_config['max_frames'] + self.feature_framerate = model_config['feature_framerate'] + self.image_resolution = 224 + if torch.cuda.is_available(): + self.device = model_config['device'] + else: + self.device = 'cpu' + self.init_model = f'{model_dir}/{ModelFile.TORCH_MODEL_BIN_FILE}' + + self.tokenizer = ClipTokenizer(model_dir) + self.rawVideoExtractor = RawVideoExtractor( + frame_rate=self.feature_framerate, size=self.image_resolution) + self.local_transform = self.rawVideoExtractor.transform + self.model = CLIP4Clip.from_pretrained( + cross_config=cross_model_config, task_config=model_config) + if hasattr(self.model, 'module'): + self.model = self.model.module.to(self.device) + else: + self.model = self.model.to(self.device) + if self.init_model: + assert exists(self.init_model) + model_state_dict = torch.load(self.init_model, map_location='cpu') + self.model.load_state_dict(model_state_dict, strict=False) + self.model.to(self.device) + + def _get_text(self, caption, tokenizer, enable_zh=False): + + if type(caption) is str: + _caption_text, s, e = caption, None, None + elif type(caption) is tuple: + if len(caption) == 3: + _caption_text, s, e = caption + elif len(caption) == 4: + _caption_text, s, e, pos = caption + else: + NotImplementedError + + if isinstance(_caption_text, list): + caption_text = random.choice(_caption_text) + else: + caption_text = _caption_text + if enable_zh: + _token = tokenizer.encode(caption_text) + input_ids = _token.ids + input_mask = _token.attention_mask + segment_ids = _token.type_ids + else: + words = tokenizer.tokenize(caption_text) + + words = [self.SPECIAL_TOKEN['CLS_TOKEN']] + words + total_length_with_CLS = self.max_words - 1 + if len(words) > total_length_with_CLS: + words = words[:total_length_with_CLS] + words = words + [self.SPECIAL_TOKEN['SEP_TOKEN']] + + input_ids = tokenizer.convert_tokens_to_ids(words) + input_mask = [1] * len(input_ids) + segment_ids = [0] * len(input_ids) + + while len(input_ids) < self.max_words: + input_ids.append(0) + input_mask.append(0) + segment_ids.append(0) + assert len(input_ids) == self.max_words + assert len(input_mask) == self.max_words + assert len(segment_ids) == self.max_words + + pairs_text = np.array(input_ids) + pairs_mask = np.array(input_mask) + pairs_segment = np.array(segment_ids) + + return pairs_text, pairs_mask, pairs_segment, s, e + + def _get_rawvideo_dec(self, + video_path, + rawVideoExtractor, + local_transform, + s=None, + e=None): + video_mask = np.zeros(self.max_frames, dtype=int) + max_video_length = 0 + + # T x 3 x H x W + video = np.zeros((self.max_frames, 3, rawVideoExtractor.size, + rawVideoExtractor.size), + dtype=float) + + if s is None: + start_time, end_time = None, None + else: + start_time = int(s) + end_time = int(e) + start_time = start_time if start_time >= 0. else 0. + end_time = end_time if end_time >= 0. else 0. + if start_time > end_time: + start_time, end_time = end_time, start_time + elif start_time == end_time: + end_time = end_time + 1 + + url_parsed = urlparse(video_path) + if url_parsed.scheme in ('file', '') and exists( + url_parsed.path): # Possibly a local file + vreader = VideoReader(video_path, ctx=cpu(0)) + else: + try: + with TemporaryDirectory() as temporary_cache_dir: + random_str = uuid.uuid4().hex + http_get_file( + url=video_path, + local_dir=temporary_cache_dir, + file_name=random_str, + cookies=None) + temp_file_path = os.path.join(temporary_cache_dir, + random_str) + vreader = VideoReader(temp_file_path, ctx=cpu(0)) + except Exception as ex: + logger.error('non video input, output is {}!!!'.format(ex)) + return video, video_mask + + fps = vreader.get_avg_fps() + f_start = 0 if start_time is None else int(start_time * fps) + f_end = int( + min(1000000000 if end_time is None else end_time * fps, + len(vreader) - 1)) + num_frames = f_end - f_start + 1 + if num_frames > 0: + # L x T x 3 x H x W + sample_fps = int(self.feature_framerate) + t_stride = int(round(float(fps) / sample_fps)) + + all_pos = list(range(f_start, f_end + 1, t_stride)) + if len(all_pos) > self.max_frames: + sample_pos = [ + all_pos[_] for _ in np.linspace( + 0, len(all_pos) - 1, num=self.max_frames, dtype=int) + ] + else: + sample_pos = all_pos + patch_images = [ + Image.fromarray(f) + for f in vreader.get_batch(sample_pos).asnumpy() + ] + patch_images = torch.stack( + [local_transform(img) for img in patch_images]) + slice_len = patch_images.shape[0] + max_video_length = max_video_length if max_video_length > slice_len else slice_len + if slice_len < 1: + pass + else: + video[:slice_len, ...] = patch_images + else: + logger.error('video path: {} error. video id: {}'.format( + video_path, video_id)) + + video_mask[:max_video_length] = [1] * max_video_length + + return video, video_mask + + def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: + + from modelscope.outputs import OutputKeys + output = {} + + if 'video' in input and input['video'] is not None: + video_path = input['video'] + video, video_mask = self._get_rawvideo_dec(video_path, + self.rawVideoExtractor, + self.local_transform) + video = torch.unsqueeze( + torch.from_numpy(video), dim=0).to(self.device) + video_mask = torch.unsqueeze( + torch.from_numpy(video_mask), dim=0).to(self.device) + + if 'text' in input and input['text'] is not None: + caption = input['text'] + pairs_text, pairs_mask, pairs_segment, s, e = self._get_text( + caption, self.tokenizer, enable_zh=False) + input_ids = torch.unsqueeze( + torch.from_numpy(pairs_text), dim=0).to(self.device) + input_mask = torch.unsqueeze( + torch.from_numpy(pairs_mask), dim=0).to(self.device) + segment_ids = torch.unsqueeze( + torch.from_numpy(pairs_segment), dim=0).to(self.device) + + phr_feat, sen_feat, obj_feat, eve_feat = self.model.get_sequence_visual_output( + input_ids, segment_ids, input_mask, video, video_mask) + + sim_espm, _, sim_oppm, _ = self.model.get_max_similarity_logits( + phr_feat, + sen_feat, + obj_feat, + eve_feat, + input_mask, + video_mask, + shaped=True) + # logger.info('sim: {}'.format(sim_espm)) + # logger.info('sim: {}'.format(sim_oppm)) + sim_tv = sim_espm + 1.5 * sim_oppm + + # logger.info('phrase prototype: {}'.format(phr_feat.shape)) + # logger.info('sentence prototype: {}'.format(sen_feat.shape)) + # logger.info('object prototype: {}'.format(obj_feat.shape)) + # logger.info('event prototype: {}'.format(eve_feat.shape)) + output[OutputKeys.TEXTVIDEO_SIM] = sim_tv.cpu().detach().numpy() + output[OutputKeys.PHRASE_PROTOTYPE] = phr_feat.cpu().detach().numpy() + output[OutputKeys.SENTENCE_PROTOTYPE] = sen_feat.cpu().detach().numpy() + output[OutputKeys.OBJECT_PROTOTYPE] = obj_feat.cpu().detach().numpy() + output[OutputKeys.EVENT_PROTOTYPE] = eve_feat.cpu().detach().numpy() + return output + + def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + return inputs diff --git a/modelscope/models/multi_modal/prost/models/tokenization_clip.py b/modelscope/models/multi_modal/prost/models/tokenization_clip.py new file mode 100644 index 00000000..97ee7156 --- /dev/null +++ b/modelscope/models/multi_modal/prost/models/tokenization_clip.py @@ -0,0 +1,161 @@ +# The implementation is adopted from the CLIP4Clip implementation, +# made pubicly available under Apache License, Version 2.0 at https://github.com/ArrowLuo/CLIP4Clip + +import gzip +import html +import os +from functools import lru_cache + +import ftfy +import regex as re + + +@lru_cache() +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a signficant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = list(range(ord('!'), + ord('~') + 1)) + list(range( + ord('¡'), + ord('¬') + 1)) + list(range(ord('®'), + ord('ÿ') + 1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8 + n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +def get_pairs(word): + """Return set of symbol pairs in a word. + Word is represented as tuple of symbols (symbols being variable-length strings). + """ + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +def basic_clean(text): + text = ftfy.fix_text(text) + text = html.unescape(html.unescape(text)) + return text.strip() + + +def whitespace_clean(text): + text = re.sub(r'\s+', ' ', text) + text = text.strip() + return text + + +class SimpleTokenizer(object): + + def __init__(self, model_dir): + bpe_path = '{}/bpe_simple_vocab_16e6.txt.gz'.format(model_dir) + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + merges = gzip.open(bpe_path).read().decode('utf-8').split('\n') + merges = merges[1:49152 - 256 - 2 + 1] + merges = [tuple(merge.split()) for merge in merges] + vocab = list(bytes_to_unicode().values()) + vocab = vocab + [v + '' for v in vocab] + for merge in merges: + vocab.append(''.join(merge)) + vocab.extend(['<|startoftext|>', '<|endoftext|>']) + self.encoder = dict(zip(vocab, range(len(vocab)))) + self.decoder = {v: k for k, v in self.encoder.items()} + self.bpe_ranks = dict(zip(merges, range(len(merges)))) + self.cache = { + '<|startoftext|>': '<|startoftext|>', + '<|endoftext|>': '<|endoftext|>' + } + self.pat = re.compile( + r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", + re.IGNORECASE) + + self.vocab = self.encoder + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token[:-1]) + (token[-1] + '', ) + pairs = get_pairs(word) + + if not pairs: + return token + '' + + while True: + bigram = min( + pairs, key=lambda pair: self.bpe_ranks.get(pair, float('inf'))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + new_word.extend(word[i:j]) + i = j + except Exception: + new_word.extend(word[i:]) + break + + if word[i] == first and i < len(word) - 1 and word[ + i + 1] == second: + new_word.append(first + second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = ' '.join(word) + self.cache[token] = word + return word + + def encode(self, text): + bpe_tokens = [] + text = whitespace_clean(basic_clean(text)).lower() + for token in re.findall(self.pat, text): + token = ''.join(self.byte_encoder[b] + for b in token.encode('utf-8')) + bpe_tokens.extend(self.encoder[bpe_token] + for bpe_token in self.bpe(token).split(' ')) + return bpe_tokens + + def decode(self, tokens): + text = ''.join([self.decoder[token] for token in tokens]) + text = bytearray([self.byte_decoder[c] for c in text]).decode( + 'utf-8', errors='replace').replace('', ' ') + return text + + def tokenize(self, text): + tokens = [] + text = whitespace_clean(basic_clean(text)).lower() + for token in re.findall(self.pat, text): + token = ''.join(self.byte_encoder[b] + for b in token.encode('utf-8')) + tokens.extend( + bpe_token for bpe_token in self.bpe(token).split(' ')) + return tokens + + def convert_tokens_to_ids(self, tokens): + return [self.encoder[bpe_token] for bpe_token in tokens] diff --git a/modelscope/models/multi_modal/prost/models/until_config.py b/modelscope/models/multi_modal/prost/models/until_config.py new file mode 100755 index 00000000..dc9753d3 --- /dev/null +++ b/modelscope/models/multi_modal/prost/models/until_config.py @@ -0,0 +1,59 @@ +# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch BERT model.""" + +from __future__ import absolute_import, division, print_function +import copy +import logging +import os +import shutil +import tarfile +import tempfile + +import json +import torch + +# from modelscope.utils.logger import get_logger +# logger = get_logger(__name__) + + +class PreCrossConfig(object): + + @classmethod + def from_dict(cls, json_object): + """Constructs a `BertConfig` from a Python dictionary of parameters.""" + config = cls(vocab_size_or_config_json_file=-1) + for key, value in json_object.items(): + config.__dict__[key] = value + return config + + @classmethod + def from_json_file(cls, json_file): + """Constructs a `BertConfig` from a json file of parameters.""" + with open(json_file, 'r', encoding='utf-8') as reader: + text = reader.read() + return cls.from_dict(json.loads(text)) + + def __repr__(self): + return str(self.to_json_string()) + + def to_dict(self): + """Serializes this instance to a Python dictionary.""" + output = copy.deepcopy(self.__dict__) + return output + + def to_json_string(self): + """Serializes this instance to a JSON string.""" + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + '\n' diff --git a/modelscope/models/multi_modal/prost/models/until_module.py b/modelscope/models/multi_modal/prost/models/until_module.py new file mode 100644 index 00000000..b33f4b77 --- /dev/null +++ b/modelscope/models/multi_modal/prost/models/until_module.py @@ -0,0 +1,574 @@ +# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch BERT model.""" + +import copy +import logging +import math + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn + +from modelscope.models.multi_modal.prost.models.until_config import \ + PreCrossConfig + + +def gelu(x): + """Implementation of the gelu activation function. + For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): + 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + """ + return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0))) + + +def swish(x): + return x * torch.sigmoid(x) + + +ACT2FN = {'gelu': gelu, 'relu': torch.nn.functional.relu, 'swish': swish} + + +class LayerNorm(nn.Module): + + def __init__(self, hidden_size, eps=1e-12): + """Construct a layernorm module in the TF style (epsilon inside the square root). + """ + super(LayerNorm, self).__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.bias = nn.Parameter(torch.zeros(hidden_size)) + self.variance_epsilon = eps + + def forward(self, x): + u = x.mean(-1, keepdim=True) + s = (x - u).pow(2).mean(-1, keepdim=True) + x = (x - u) / torch.sqrt(s + self.variance_epsilon) + return self.weight * x + self.bias + + +class CrossEn(nn.Module): + + def __init__(self, config=None): + super(CrossEn, self).__init__() + + def forward(self, sim_matrix): + logpt = F.log_softmax(sim_matrix, dim=-1) + logpt = torch.diag(logpt) + nce_loss = -logpt + sim_loss = nce_loss.mean() + return sim_loss + + +class AllGather(torch.autograd.Function): + """An autograd function that performs allgather on a tensor.""" + + @staticmethod + def forward(ctx, tensor, args): + if args.world_size == 1: + ctx.rank = args.local_rank + ctx.batch_size = tensor.shape[0] + return tensor + else: + output = [torch.empty_like(tensor) for _ in range(args.world_size)] + torch.distributed.all_gather(output, tensor) + ctx.rank = args.local_rank + ctx.batch_size = tensor.shape[0] + return torch.cat(output, dim=0) + + @staticmethod + def backward(ctx, grad_output): + return ( + grad_output[ctx.batch_size * ctx.rank:ctx.batch_size + * (ctx.rank + 1)], + None, + ) + + +class AllGather2(torch.autograd.Function): + """An autograd function that performs allgather on a tensor.""" + # https://github.com/PyTorchLightning/lightning-bolts/blob/8d3fbf7782e3d3937ab8a1775a7092d7567f2933/pl_bolts/models/self_supervised/simclr/simclr_module.py#L20 + @staticmethod + def forward(ctx, tensor, args): + if args.world_size == 1: + ctx.rank = args.local_rank + ctx.batch_size = tensor.shape[0] + return tensor + else: + output = [torch.empty_like(tensor) for _ in range(args.world_size)] + torch.distributed.all_gather(output, tensor) + ctx.rank = args.local_rank + ctx.batch_size = tensor.shape[0] + return torch.cat(output, dim=0) + + @staticmethod + def backward(ctx, grad_output): + grad_input = grad_output.clone() + torch.distributed.all_reduce( + grad_input, op=torch.distributed.ReduceOp.SUM, async_op=False) + return (grad_input[ctx.rank * ctx.batch_size:(ctx.rank + 1) + * ctx.batch_size], None) + + +class PreTrainedModel(nn.Module): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + + def __init__(self, config, *inputs, **kwargs): + super(PreTrainedModel, self).__init__() + if not isinstance(config, PreCrossConfig): + raise ValueError( + 'Parameter config in `{}(config)` should be an instance of class `PreCrossConfig`. ' + 'To create a model from a Google pretrained model use ' + '`model = {}.from_pretrained(PRETRAINED_MODEL_NAME)`'.format( + self.__class__.__name__, self.__class__.__name__)) + self.config = config + + def init_weights(self, module): + """ Initialize the weights. + """ + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_( + mean=0.0, std=self.config.initializer_range) + elif isinstance(module, LayerNorm): + if 'beta' in dir(module) and 'gamma' in dir(module): + module.beta.data.zero_() + module.gamma.data.fill_(1.0) + else: + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + + def resize_token_embeddings(self, new_num_tokens=None): + raise NotImplementedError + + @classmethod + def init_preweight(cls, model, state_dict, prefix=None, task_config=None): + old_keys = [] + new_keys = [] + for key in state_dict.keys(): + new_key = None + if 'gamma' in key: + new_key = key.replace('gamma', 'weight') + if 'beta' in key: + new_key = key.replace('beta', 'bias') + if new_key: + old_keys.append(key) + new_keys.append(new_key) + for old_key, new_key in zip(old_keys, new_keys): + state_dict[new_key] = state_dict.pop(old_key) + + if prefix is not None: + old_keys = [] + new_keys = [] + for key in state_dict.keys(): + old_keys.append(key) + new_keys.append(prefix + key) + for old_key, new_key in zip(old_keys, new_keys): + state_dict[new_key] = state_dict.pop(old_key) + + missing_keys = [] + unexpected_keys = [] + error_msgs = [] + # copy state_dict so _load_from_state_dict can modify it + metadata = getattr(state_dict, '_metadata', None) + state_dict = state_dict.copy() + if metadata is not None: + state_dict._metadata = metadata + + def load(module, prefix=''): + local_metadata = {} if metadata is None else metadata.get( + prefix[:-1], {}) + module._load_from_state_dict(state_dict, prefix, local_metadata, + True, missing_keys, unexpected_keys, + error_msgs) + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + '.') + + load(model, prefix='') + + # if prefix is None and (task_config is None or task_config.local_rank == 0): + # logger.info("-" * 20) + # if len(missing_keys) > 0: + # logger.info("Weights of {} not initialized from pretrained model: {}" + # .format(model.__class__.__name__, "\n " + "\n ".join(missing_keys))) + # if len(unexpected_keys) > 0: + # logger.info("Weights from pretrained model not used in {}: {}" + # .format(model.__class__.__name__, "\n " + "\n ".join(unexpected_keys))) + # if len(error_msgs) > 0: + # logger.error("Weights from pretrained model cause errors in {}: {}" + # .format(model.__class__.__name__, "\n " + "\n ".join(error_msgs))) + + return model + + @property + def dtype(self): + """ + :obj:`torch.dtype`: The dtype of the module (assuming that all the module parameters have the same dtype). + """ + try: + return next(self.parameters()).dtype + except StopIteration: + # For nn.DataParallel compatibility in PyTorch 1.5 + def find_tensor_attributes(module: nn.Module): + tuples = [(k, v) for k, v in module.__dict__.items() + if torch.is_tensor(v)] + return tuples + + gen = self._named_members(get_members_fn=find_tensor_attributes) + first_tuple = next(gen) + return first_tuple[1].dtype + + @classmethod + def from_pretrained(cls, config, state_dict=None, *inputs, **kwargs): + """ + Instantiate a PreTrainedModel from a pre-trained model file or a pytorch state dict. + Download and cache the pre-trained model file if needed. + """ + # Instantiate model. + model = cls(config, *inputs, **kwargs) + if state_dict is None: + return model + model = cls.init_preweight(model, state_dict) + + return model + + +class PatchShiftModule(nn.Module): + + def __init__(self, net, video_frame, n_div): + super().__init__() + self.net = net + self.video_frame = video_frame + self.n_div = n_div + + def forward(self, + query, + key, + value, + key_padding_mask=None, + need_weights=True, + attn_mask=None): + # here q == k == v, psm means patch shift output + x = query # shape here is LND, not NLD (50, 384, 768) + x = x.permute(1, 0, 2) # LND -> NLD + patch_len = x.shape[-2] + fold = patch_len // self.n_div + x = x.reshape(-1, self.video_frame, x.shape[-2], + x.shape[-1]) # shape = [bs, frame, grid ** 2, width] + psm = torch.zeros_like(x) # shape = [bs, frame, grid ** 2, width] + psm[:, :, :, :] = x[:, :, :, :] + lshift_indices = torch.arange(start=1, end=patch_len, step=fold) + psm[:, 1:, lshift_indices, :] = x[:, :-1, + lshift_indices, :] # f_t = f_t-1 + rshift_indices = torch.arange(start=1 + 3, end=patch_len, step=fold) + psm[:, :-1, rshift_indices, :] = x[:, 1:, + rshift_indices, :] # f_t = f_t+1 + x = psm.reshape(-1, patch_len, x.shape[-1]) + x = x.permute(1, 0, 2) # NLD -> LND + + return self.net( + x, x, x, need_weights=need_weights, attn_mask=attn_mask) + + +def make_patch_shift(net, video_frame=12, shift_layers=4, n_div=7): + ''' + Args: + net: CLIP + video_frame: need predefine here + shift_layers: layers to be shift + ''' + + def make_trans_patch_shift(stage, shift_layers): + blocks = list(stage.children()) + for i, b in enumerate(blocks): + if i >= 10 and i <= 11: + blocks[i].attn = PatchShiftModule( + b.attn, video_frame=video_frame, n_div=n_div) + return nn.Sequential(*blocks) + + net.clip.visual.transformer.resblocks = make_trans_patch_shift( + net.clip.visual.transformer.resblocks, shift_layers=shift_layers) + + +def _get_clones(module, N): + return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) + + +class Event_Layer(nn.Module): + + def __init__(self, + d_model, + nhead, + dim_feedforward=2048, + dropout=0.1, + activation='relu', + normalize_before=False, + is_weights=False): + super().__init__() + self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) + self.self_attn_vis = nn.MultiheadAttention( + d_model, nhead, dropout=dropout) + self.multihead_attn = nn.MultiheadAttention( + d_model, nhead, dropout=dropout) + # Implementation of Feedforward model + self.linear1 = nn.Linear(d_model, dim_feedforward) + self.dropout = nn.Dropout(dropout) + self.linear2 = nn.Linear(dim_feedforward, d_model) + + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + self.norm3 = nn.LayerNorm(d_model) + self.norm4 = nn.LayerNorm(d_model) + self.norm5 = nn.LayerNorm(d_model) + + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(dropout) + self.dropout3 = nn.Dropout(dropout) + + self.activation = nn.ReLU(inplace=True) + self.normalize_before = normalize_before + self.is_weights = is_weights + + def forward(self, tgt, memory, pos=None, query_pos=None): + + tgt = self.norm1(tgt) + memory = self.norm2(memory) + tgt = self.self_attn(tgt, tgt, tgt)[0] + tgt = self.norm3(tgt) + + tgt2, atten_weights = self.multihead_attn(tgt, memory, memory) + tgt = tgt + self.dropout1(tgt2) + tgt = self.norm4(tgt) + tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt)))) + tgt = tgt + self.dropout2(tgt2) + tgt = self.norm5(tgt) + + return tgt, atten_weights + + +def adaptive_mask(aa, bb, ada_para): + tensor = torch.zeros((aa, bb)) + adaptive_num = int(bb * ada_para) + cc = int(bb / aa) + for i in range(aa): + start_col = i * cc + end_col = start_col + cc + adaptive_num + if end_col > bb - 1: + tmp = end_col - (bb - 1) + start_col = start_col - tmp + if start_col < 0: + start_col = 0 + end_col = bb + tensor[i, start_col:end_col] = 1 + tensor = ~tensor.bool() + return tensor + + +class Frame_Layer(nn.Module): + + def __init__(self, + d_model, + nhead, + dim_feedforward=2048, + para=1.0, + dropout=0.1, + activation='relu', + normalize_before=False, + is_weights=False): + super().__init__() + self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) + self.self_attn_vis = nn.MultiheadAttention( + d_model, nhead, dropout=dropout) + self.multihead_attn = nn.MultiheadAttention( + d_model, nhead, dropout=dropout) + # Implementation of Feedforward model + self.linear1 = nn.Linear(d_model, dim_feedforward) + self.dropout = nn.Dropout(dropout) + self.linear2 = nn.Linear(dim_feedforward, d_model) + + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + self.norm3 = nn.LayerNorm(d_model) + self.norm4 = nn.LayerNorm(d_model) + + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(dropout) + + self.activation = nn.ReLU(inplace=True) + self.normalize_before = normalize_before + self.is_weights = is_weights + self.mask_para = para + + def forward(self, tgt, memory, pos=None, query_pos=None): + tgt = self.norm1(tgt) + memory = self.norm2(memory) + mask_new = adaptive_mask(tgt.shape[0], memory.shape[0], ada_para=0.2) + tgt2, atten_weights = self.multihead_attn( + tgt, memory, memory, attn_mask=mask_new.cuda()) + tgt = tgt + self.dropout1(tgt2) + + tgt = self.norm3(tgt) + tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt)))) + tgt = tgt + self.dropout2(tgt2) + tgt = self.norm4(tgt) + + return tgt, atten_weights + + +class TransDecoder(nn.Module): + + def __init__(self, + decoder_layer, + num_layers, + norm=None, + return_intermediate=False): + super().__init__() + self.layers = _get_clones(decoder_layer, num_layers) + self.num_layers = num_layers + self.norm = norm + self.return_intermediate = return_intermediate + + def forward(self, tgt, memory, pos=None, query_pos=None): + output = tgt + + intermediate = [] + all_weights = [] + + for layer in self.layers: + output, weights = layer( + output, memory, pos=pos, query_pos=query_pos) + if self.return_intermediate: + intermediate.append(self.norm(output)) + all_weights.append(weights) + + if self.norm is not None: + output = self.norm(output) + if self.return_intermediate: + intermediate.pop() + intermediate.append(output) + + if self.return_intermediate: + return torch.stack(intermediate), torch.stack(all_weights) + return output.unsqueeze(0) + + +class Event_decoder(nn.Module): + + def __init__(self, + num_attris=3, + layers=1, + heads=1, + dim_ftr=512, + pos_emb=False, + length=1, + dim_feedforward=512, + without_init=False): + super().__init__() + embedding_dim = dim_ftr + + d_model = dim_ftr + dim_feedforward = dim_feedforward + + self.V = nn.Parameter( + torch.Tensor(num_attris, dim_feedforward), requires_grad=True) + nn.init.xavier_uniform_(self.V) + decoder_layer = Event_Layer( + d_model=d_model, nhead=heads, dim_feedforward=dim_feedforward) + self.event_decoder = TransDecoder( + decoder_layer, + layers, + nn.LayerNorm(d_model), + return_intermediate=True) + self.use_pos_enc = pos_emb + + if self.use_pos_enc: + self.position_encoding_pre = positionalencoding2d( + embedding_dim, 14, 14).unsqueeze(0) + + def forward(self, features): + batch_size = features.shape[0] + if self.use_pos_enc: # False + pos_encoding = self.position_encoding_pre( + features, + torch.zeros(features.shape[0], 14, 14, + dtype=torch.bool).cuda()) + features = features + pos_encoding + + enco_others = features.permute(1, 0, 2) + h_attr = self.V + h_attr_batch = h_attr.unsqueeze(0).repeat(batch_size, 1, 1) + h_attr_batch = h_attr_batch.permute(1, 0, 2) + + hs, _ = self.event_decoder(h_attr_batch, enco_others) + hs = hs[-1].permute(1, 0, 2) + return hs + + +class Frame_decoder(nn.Module): + + def __init__(self, + num_attris=3, + layers=1, + heads=1, + dim_ftr=512, + pos_emb=False, + length=1, + dim_feedforward=512, + without_init=False): + super().__init__() + embedding_dim = dim_ftr + d_model = dim_ftr + dim_feedforward = dim_feedforward + + self.V = nn.Parameter( + torch.Tensor(num_attris, dim_feedforward), requires_grad=True) + nn.init.xavier_uniform_(self.V) + decoder_layer = Frame_Layer( + d_model=d_model, nhead=heads, dim_feedforward=dim_feedforward) + self.event_decoder = TransDecoder( + decoder_layer, + layers, + nn.LayerNorm(d_model), + return_intermediate=True) + self.use_pos_enc = pos_emb + + if self.use_pos_enc: + self.position_encoding_pre = positionalencoding2d( + embedding_dim, 14, 14).unsqueeze(0) + + def forward(self, features): + batch_size = features.shape[0] + if self.use_pos_enc: + pos_encoding = self.position_encoding_pre( + features, + torch.zeros(features.shape[0], 14, 14, + dtype=torch.bool).cuda()) + features = features + pos_encoding + + enco_others = features.permute(1, 0, 2) + h_attr = self.V + h_attr_batch = h_attr.unsqueeze(0).repeat(batch_size, 1, 1) + h_attr_batch = h_attr_batch.permute(1, 0, 2) + + hs, _ = self.event_decoder(h_attr_batch, enco_others) + hs = hs[-1].permute(1, 0, 2) + + return hs diff --git a/modelscope/outputs/outputs.py b/modelscope/outputs/outputs.py index a6d91711..f6ef0f6b 100644 --- a/modelscope/outputs/outputs.py +++ b/modelscope/outputs/outputs.py @@ -48,6 +48,11 @@ class OutputKeys(object): PROBABILITIES = 'probabilities' DIALOG_STATES = 'dialog_states' VIDEO_EMBEDDING = 'video_embedding' + PHRASE_PROTOTYPE = 'phrase_prototype' + OBJECT_PROTOTYPE = 'object_prototype' + SENTENCE_PROTOTYPE = 'sentence_prototype' + EVENT_PROTOTYPE = 'event_prototype' + TEXTVIDEO_SIM = 'textvideo_sim' UUID = 'uuid' WORD = 'word' KWS_LIST = 'kws_list' @@ -106,6 +111,11 @@ OutputTypes = { OutputKeys.PROBABILITIES: np.ndarray, OutputKeys.DIALOG_STATES: object, OutputKeys.VIDEO_EMBEDDING: np.ndarray, + OutputKeys.PHRASE_PROTOTYPE: np.ndarray, + OutputKeys.OBJECT_PROTOTYPE: np.ndarray, + OutputKeys.SENTENCE_PROTOTYPE: np.ndarray, + OutputKeys.EVENT_PROTOTYPE: np.ndarray, + OutputKeys.TEXTVIDEO_SIM: np.ndarray, OutputKeys.UUID: str, OutputKeys.WORD: str, OutputKeys.KWS_LIST: List[str], @@ -329,6 +339,24 @@ OutputTypeSchema = { 'type': 'number' } }, + OutputKeys.PHRASE_PROTOTYPE: { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, + OutputKeys.OBJECT_PROTOTYPE: { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, + OutputKeys.TEXTVIDEO_SIM: { + 'type': 'array', + 'items': { + 'type': 'number' + } + }, OutputKeys.UUID: { 'type': 'string' }, @@ -916,6 +944,32 @@ TASK_OUTPUTS = { # } Tasks.video_embedding: [OutputKeys.VIDEO_EMBEDDING], + # phrase prototype result for single sentence + # { + # "phrase_prototype": np.array with shape [K*D], + # } + # sentence prototype result for single sentence + # { + # "sentence_prototype": np.array with shape [1*D], + # } + # object prototype result for single video + # { + # "object_prototype": np.array with shape [N*K*D], + # } + # event prototype result for single video + # { + # "event_prototype": np.array with shape [N*M*D], + # } + # text search video result for single sentence + # { + # "textvideo_sim": np.array with shape [N*N], + # } + Tasks.text_video_retrieval: [ + OutputKeys.PHRASE_PROTOTYPE, OutputKeys.SENTENCE_PROTOTYPE, + OutputKeys.OBJECT_PROTOTYPE, OutputKeys.EVENT_PROTOTYPE, + OutputKeys.TEXTVIDEO_SIM + ], + # video stabilization task result for a single video # {"output_video": "path_to_rendered_video"} Tasks.video_stabilization: [OutputKeys.OUTPUT_VIDEO], diff --git a/modelscope/pipeline_inputs.py b/modelscope/pipeline_inputs.py index 38403d1d..712f6ff9 100644 --- a/modelscope/pipeline_inputs.py +++ b/modelscope/pipeline_inputs.py @@ -418,6 +418,10 @@ TASK_INPUTS = { 'img': InputType.IMAGE, 'text': InputType.TEXT }, + Tasks.text_video_retrieval: { + 'video': InputType.VIDEO, + 'text': InputType.TEXT + }, Tasks.visual_question_answering: { 'image': InputType.IMAGE, 'text': InputType.TEXT diff --git a/modelscope/pipelines/multi_modal/__init__.py b/modelscope/pipelines/multi_modal/__init__.py index 11c28fdd..25e45667 100644 --- a/modelscope/pipelines/multi_modal/__init__.py +++ b/modelscope/pipelines/multi_modal/__init__.py @@ -4,30 +4,39 @@ from typing import TYPE_CHECKING from modelscope.utils.import_utils import LazyImportModule if TYPE_CHECKING: - from .generative_multi_modal_embedding_pipeline import GEMMMultiModalEmbeddingPipeline + from .asr_pipeline import AutomaticSpeechRecognitionPipeline + from .diffusers_wrapped import (ChineseStableDiffusionPipeline, + StableDiffusionPipeline) + from .document_vl_embedding_pipeline import DocumentVLEmbeddingPipeline + from .generative_multi_modal_embedding_pipeline import \ + GEMMMultiModalEmbeddingPipeline from .image_captioning_pipeline import ImageCaptioningPipeline - from .visual_entailment_pipeline import VisualEntailmentPipeline - from .visual_grounding_pipeline import VisualGroundingPipeline + from .mgeo_ranking_pipeline import MGeoRankingPipeline from .multi_modal_embedding_pipeline import MultiModalEmbeddingPipeline + from .multimodal_dialogue_pipeline import MultimodalDialoguePipeline + from .prost_text_video_retrieval_pipeline import \ + ProSTTextVideoRetrievalPipeline + from .soonet_video_temporal_grounding_pipeline import \ + SOONetVideoTemporalGroundingPipeline from .text_to_image_synthesis_pipeline import TextToImageSynthesisPipeline + from .text_to_video_synthesis_pipeline import TextToVideoSynthesisPipeline + from .video_captioning_pipeline import VideoCaptioningPipeline from .video_multi_modal_embedding_pipeline import \ VideoMultiModalEmbeddingPipeline - from .visual_question_answering_pipeline import VisualQuestionAnsweringPipeline - from .asr_pipeline import AutomaticSpeechRecognitionPipeline - from .mgeo_ranking_pipeline import MGeoRankingPipeline - from .document_vl_embedding_pipeline import DocumentVLEmbeddingPipeline - from .video_captioning_pipeline import VideoCaptioningPipeline - from .video_question_answering_pipeline import VideoQuestionAnsweringPipeline - from .diffusers_wrapped import StableDiffusionPipeline, ChineseStableDiffusionPipeline - from .soonet_video_temporal_grounding_pipeline import SOONetVideoTemporalGroundingPipeline - from .text_to_video_synthesis_pipeline import TextToVideoSynthesisPipeline - from .multimodal_dialogue_pipeline import MultimodalDialoguePipeline + from .video_question_answering_pipeline import \ + VideoQuestionAnsweringPipeline + from .visual_entailment_pipeline import VisualEntailmentPipeline + from .visual_grounding_pipeline import VisualGroundingPipeline + from .visual_question_answering_pipeline import \ + VisualQuestionAnsweringPipeline else: _import_structure = { 'image_captioning_pipeline': ['ImageCaptioningPipeline'], 'visual_entailment_pipeline': ['VisualEntailmentPipeline'], 'visual_grounding_pipeline': ['VisualGroundingPipeline'], 'multi_modal_embedding_pipeline': ['MultiModalEmbeddingPipeline'], + 'prost_text_video_retrieval_pipeline': + ['ProSTTextVideoRetrievalPipeline'], 'text_to_image_synthesis_pipeline': ['TextToImageSynthesisPipeline'], 'visual_question_answering_pipeline': ['VisualQuestionAnsweringPipeline'], diff --git a/modelscope/pipelines/multi_modal/prost_text_video_retrieval_pipeline.py b/modelscope/pipelines/multi_modal/prost_text_video_retrieval_pipeline.py new file mode 100644 index 00000000..ecb27a7f --- /dev/null +++ b/modelscope/pipelines/multi_modal/prost_text_video_retrieval_pipeline.py @@ -0,0 +1,56 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +from typing import Any, Dict + +from modelscope.metainfo import Pipelines +from modelscope.pipelines.base import Input, Pipeline +from modelscope.pipelines.builder import PIPELINES +from modelscope.utils.constant import Tasks +from modelscope.utils.device import device_placement +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +@PIPELINES.register_module( + Tasks.text_video_retrieval, + module_name=Pipelines.prost_text_video_retrieval) +class ProSTTextVideoRetrievalPipeline(Pipeline): + ''' + https://www.modelscope.cn/models/damo/multi_modal_clip_vtretrieval_prost/summary + + from modelscope.pipelines import pipeline + from modelscope.utils.constant import Tasks + text_video_retrieval= pipeline( + Tasks.text_video_retrieval, + model='damo/multi_modal_clip_vtretrieval_prost') + video_path = 'https://modelscope.oss-cn-beijing.aliyuncs.com/test/videos/multi_modal_test_video_9770.mp4' + caption = 'a person is connecting something to system' + _input = {'video': video_path, 'text': caption} + result = text_video_retrieval(_input) + ''' + + def __init__(self, model: str, **kwargs): + """ + use `model` to create a text_video_retrieval pipeline for prediction + Args: + model: model id on modelscope hub. + """ + super().__init__(model=model) + self.model.eval() + + def preprocess(self, input: Input) -> Dict[str, Any]: + return input + + def _process_single(self, input: Input, *args, **kwargs) -> Dict[str, Any]: + with device_placement(self.framework, self.device_name): + out = self.forward(input) + + self._check_output(out) + return out + + def forward(self, input: Dict[str, Any]) -> Dict[str, Any]: + return self.model(input) + + def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + return inputs diff --git a/modelscope/utils/constant.py b/modelscope/utils/constant.py index d540f665..b9b6f811 100644 --- a/modelscope/utils/constant.py +++ b/modelscope/utils/constant.py @@ -243,6 +243,7 @@ class MultiModalTasks(object): visual_grounding = 'visual-grounding' text_to_image_synthesis = 'text-to-image-synthesis' multi_modal_embedding = 'multi-modal-embedding' + text_video_retrieval = 'text-video-retrieval' generative_multi_modal_embedding = 'generative-multi-modal-embedding' multi_modal_similarity = 'multi-modal-similarity' visual_question_answering = 'visual-question-answering' diff --git a/tests/pipelines/test_prost_text_video_retrieval.py b/tests/pipelines/test_prost_text_video_retrieval.py new file mode 100644 index 00000000..169c7369 --- /dev/null +++ b/tests/pipelines/test_prost_text_video_retrieval.py @@ -0,0 +1,42 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import unittest + +# import modelscope +from modelscope.pipelines import pipeline +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger +from modelscope.utils.test_utils import test_level + +logger = get_logger() + + +class ProSTTextVideoRetrievalTest(unittest.TestCase): + + def setUp(self) -> None: + self.task = Tasks.text_video_retrieval + self.model_id = 'damo/multi_modal_clip_vtretrieval_prost' + + video_path = 'https://modelscope.oss-cn-beijing.aliyuncs.com/test/videos/multi_modal_test_video_9770.mp4' + caption = 'a person is connecting something to system' + # caption = 'a dog and a cat are friends' + _input = {'video': video_path, 'text': caption} + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run(self): + pipeline_prost_text_video_retrieval = pipeline( + Tasks.text_video_retrieval, model=self.model_id) + output = pipeline_prost_text_video_retrieval(self._input) + logger.info('t2v sim: {}'.format(output['textvideo_sim'])) + logger.info('phrase prototype: {}'.format( + output['phrase_prototype'].shape)) + logger.info('object prototype: {}'.format( + output['object_prototype'].shape)) + logger.info('sentence prototype: {}'.format( + output['sentence_prototype'].shape)) + logger.info('event prototype: {}'.format( + output['event_prototype'].shape)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/run_config.yaml b/tests/run_config.yaml index 84d8d2ad..4c4d732f 100644 --- a/tests/run_config.yaml +++ b/tests/run_config.yaml @@ -45,6 +45,7 @@ isolated: # test cases that may require excessive anmount of GPU memory or run - test_table_recognition.py - test_conversational_text_to_sql.py - test_video_multi_modal_embedding.py + - test_prost_text_video_retrieval.py - test_image_skychange.py - test_video_stabilization.py - test_video_super_resolution.py