[to #42322933] Add vldoc to maas lib

Test
```python
python tests/pipelines/test_document_vl_embedding.py
```
Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/11199555

modelscope/metainfo.py

@@ -127,6 +127,7 @@ class Models(object):
     multi_stage_diffusion = 'multi-stage-diffusion-text-to-image-synthesis'
     team = 'team-multi-modal-similarity'
     video_clip = 'video-clip-multi-modal-embedding'
+    vldoc = 'vldoc'
     hitea = 'hitea'
 
     # science models
@@ -336,6 +337,7 @@ class Pipelines(object):
     image_text_retrieval = 'image-text-retrieval'
     ofa_ocr_recognition = 'ofa-ocr-recognition'
     ofa_asr = 'ofa-asr'
+    document_vl_embedding = 'document-vl-embedding'
     video_captioning = 'video-captioning'
     video_question_answering = 'video-question-answering'
 
@@ -464,6 +466,7 @@ class Preprocessors(object):
     ofa_tasks_preprocessor = 'ofa-tasks-preprocessor'
     clip_preprocessor = 'clip-preprocessor'
     mplug_tasks_preprocessor = 'mplug-tasks-preprocessor'
+    vldoc_preprocessor = 'vldoc-preprocessor'
     hitea_tasks_preprocessor = 'hitea-tasks-preprocessor'
 
     # science preprocessor

modelscope/models/multi_modal/__init__.py

@@ -16,6 +16,7 @@ if TYPE_CHECKING:
         OfaForTextToImageSynthesis
     from .multi_stage_diffusion import \
         MultiStageDiffusionForTextToImageSynthesis
+    from .vldoc import VLDocForDocVLEmbedding
 
 else:
     _import_structure = {
@@ -29,7 +30,8 @@ else:
         'ofa_for_text_to_image_synthesis_model':
         ['OfaForTextToImageSynthesis'],
         'multi_stage_diffusion':
-        ['MultiStageDiffusionForTextToImageSynthesis']
+        ['MultiStageDiffusionForTextToImageSynthesis'],
+        'vldoc': ['VLDocForDocVLEmbedding'],
     }
 
     import sys

modelscope/models/multi_modal/vldoc/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from .model import VLDocForDocVLEmbedding

modelscope/models/multi_modal/vldoc/conv_fpn_trans.py

@@ -0,0 +1,293 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import random
+from collections import OrderedDict
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from timm.models.layers import DropPath, trunc_normal_
+
+from modelscope.models.multi_modal.vldoc.convnext import convnext_tiny
+from modelscope.utils.logger import get_logger
+
+try:
+    import apex
+    import apex.normalization
+    LN = apex.normalization.FusedLayerNorm
+except ImportError:
+    LN = torch.nn.LayerNorm
+
+logging = get_logger()
+
+
+class ResidualAttentionBlock(nn.Module):
+
+    def __init__(self,
+                 d_model: int,
+                 n_head: int,
+                 attn_mask: torch.Tensor = None,
+                 expand_ratio=4.0,
+                 init_values: float = None):
+        """
+        The implementation of the transformer block refers to:
+        https://github.com/openai/CLIP/blob/b46f5ac7587d2e1862f8b7b1573179d80dcdd620/clip/model.py
+        """
+        super().__init__()
+
+        self.attn = nn.MultiheadAttention(d_model, n_head)
+        self.ln_1 = LN(d_model)
+        self.mlp = nn.Sequential(
+            OrderedDict([('c_fc', nn.Linear(d_model, d_model * expand_ratio)),
+                         ('gelu', QuickGELU()),
+                         ('c_proj', nn.Linear(d_model * expand_ratio,
+                                              d_model))]))
+        self.ln_2 = LN(d_model)
+        self.attn_mask = attn_mask
+        if init_values is not None:
+            self.gamma_1 = nn.Parameter(
+                init_values * torch.ones((d_model)), requires_grad=True)
+            self.gamma_2 = nn.Parameter(
+                init_values * torch.ones((d_model)), requires_grad=True)
+        else:
+            self.gamma_1, self.gamma_2 = 1.0, 1.0
+
+    def attention(self, x: torch.Tensor):
+        self.attn_mask = self.attn_mask.to(
+            dtype=x.dtype,
+            device=x.device) if self.attn_mask is not None else None
+        return self.attn(
+            x, x, x, need_weights=False, attn_mask=self.attn_mask)[0]
+
+    def forward(self, x: torch.Tensor):
+        x = x + self.gamma_1 * self.attention(self.ln_1(x))
+        x = x + self.gamma_2 * self.mlp(self.ln_2(x))
+        return x
+
+
+class QuickGELU(nn.Module):
+
+    def forward(self, x: torch.Tensor):
+        return x * torch.sigmoid(1.702 * x)
+
+
+def drop_grid(grid_map, drop_range=(0.3, 0.8), training=False):
+    """
+    only drop in the training phase.
+    grid_map: [N, D, T1, ...]
+    """
+    if training:
+        drop_ratio = random.random() * (drop_range[1]
+                                        - drop_range[0]) + drop_range[0]
+        # [N, T1, ...], True will be dropped
+        mask = (torch.rand_like(grid_map[:, 0]) < drop_ratio).bool()
+        grid_map = grid_map.masked_fill(mask.unsqueeze(1), 0.0)
+    return grid_map
+
+
+class GumbelSample(nn.Module):
+
+    def __init__(self, in_dim, num_keep):
+        super(GumbelSample, self).__init__()
+        self.keep_layer = nn.Sequential(
+            nn.Conv2d(
+                in_dim, 256, 3, stride=1, padding=3, dilation=3, bias=False),
+            nn.BatchNorm2d(256),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(
+                256, 256, 3, stride=1, padding=2, dilation=2, bias=False),
+            nn.BatchNorm2d(256),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(256, 1, 3, stride=1, padding=1, dilation=1),
+            nn.Sigmoid(),
+        )
+        self.num_keep = num_keep
+        self.diffusion = nn.Conv2d(in_dim, in_dim, 3, padding=1)
+        self.dropout = nn.Dropout(0.1)
+
+    def forward(self, x, tau=1):
+        """
+        x: [N, C, H, W]
+        """
+        N = x.size(0)
+        keep_score = self.keep_layer(x)
+        keep_score = torch.clamp(keep_score, min=0.0, max=1.0)
+        keep_score = torch.cat([keep_score, 1 - keep_score],
+                               dim=1) + 1e-5  # [N, 2, H, W]
+        gumbel_score = F.gumbel_softmax(
+            keep_score.log(), tau=tau, hard=False, dim=1)
+        # differentiable hard mode
+        index = gumbel_score.max(dim=1, keepdim=True)[1]
+        gumbel_hard = torch.zeros_like(
+            gumbel_score,
+            memory_format=torch.legacy_contiguous_format).scatter_(
+                1, index, 1.0)
+        gumbel_hard = gumbel_hard - gumbel_score.detach() + gumbel_score
+        #
+        gumbel_score = gumbel_score[:, 0].contiguous().view(N,
+                                                            -1)  # [N, H x W]
+        gumbel_hard = gumbel_hard[:, 0].contiguous().view(N, -1)
+        # sort by score
+        idx_true = torch.topk(
+            gumbel_score, self.num_keep, dim=1)[1]  # [N, num_keep]
+        topk_mask = torch.zeros_like(gumbel_score).bool().fill_(
+            False).scatter_(1, idx_true, True)  # [N, H x W]
+        return topk_mask, gumbel_hard, keep_score[:, 0]
+
+    def sample(self, x, topk_mask, gumbel_hard):
+        N, D, H, W = x.size()
+        x = x.contiguous().view(N, D, -1)  # [N, D, HxW]
+        x = x * gumbel_hard.unsqueeze(1)
+        x = x.transpose(1, 2)  # [N, HxW, D]
+        x = x[topk_mask].contiguous().view(N, -1, D)  # [N, num_keep, D]
+        x = drop_grid(
+            x.transpose(1, 2), drop_range=(0.0, 0.2),
+            training=self.training).transpose(1, 2)
+        return x
+
+    def random_sample(self, x):
+        N, D, H, W = x.size()
+        x = x.contiguous().view(N, D, -1)  # [N, D, HxW]
+        x = x.transpose(1, 2)  # [N, HxW, D]
+        # generate random mask
+        idx_true = torch.topk(
+            torch.rand_like(x[:, :, 0]), self.num_keep,
+            dim=1)[1]  # [N, num_keep]
+        topk_mask = torch.zeros_like(x[:, :, 0]).bool().fill_(False).scatter_(
+            1, idx_true, True)  # [N, H x W]
+        # apply the mask
+        x = x[topk_mask].contiguous().view(N, -1, D)  # [N, num_keep, D]
+        x = drop_grid(
+            x.transpose(1, 2), drop_range=(0.0, 0.2),
+            training=self.training).transpose(1, 2)
+        return x, topk_mask
+
+    def restore(self, x, topk_mask, src):
+        """
+        x: [N, D, H, W]
+        topk_mask: [N, HxW]
+        src: [N, num_keep, D]
+        """
+        N, D, H, W = x.size()
+        x = drop_grid(x, drop_range=(0.2, 0.8), training=self.training)
+        x = x.contiguous().view(N, D, -1).transpose(1, 2)  # [N, HxW, D]
+        x = x.masked_scatter(topk_mask.unsqueeze(-1), src)
+        x = x.transpose(1, 2).contiguous().view(N, D, H, W)
+        x = self.dropout(self.diffusion(x))
+        return x
+
+
+class FPNTrans(nn.Module):
+
+    def __init__(self,
+                 trans_layers=2,
+                 inner_channels=256,
+                 img_size=(896, 896),
+                 inner_vit=False,
+                 out_sampling=False):
+        super(FPNTrans, self).__init__()
+        self.cnn = convnext_tiny(pretrained=True, in_22k=True)
+        self.dims = self.cnn.dims
+        self.img_size = img_size
+        # FPN in DB
+        self.up5 = nn.Upsample(scale_factor=2, mode='nearest')
+        self.up4 = nn.Upsample(scale_factor=2, mode='nearest')
+        self.up3 = nn.Upsample(scale_factor=2, mode='nearest')
+
+        self.in5 = nn.Conv2d(self.dims[-1], inner_channels, 1, bias=False)
+        self.in4 = nn.Conv2d(self.dims[-2], inner_channels, 1, bias=False)
+        self.in3 = nn.Conv2d(self.dims[-3], inner_channels, 1, bias=False)
+        self.in2 = nn.Conv2d(self.dims[-4], inner_channels, 1, bias=False)
+
+        self.out5 = nn.Sequential(
+            nn.Conv2d(
+                inner_channels, inner_channels // 4, 3, padding=1, bias=False),
+            nn.Upsample(scale_factor=8, mode='nearest'))
+        self.out4 = nn.Sequential(
+            nn.Conv2d(
+                inner_channels, inner_channels // 4, 3, padding=1, bias=False),
+            nn.Upsample(scale_factor=4, mode='nearest'))
+        self.out3 = nn.Sequential(
+            nn.Conv2d(
+                inner_channels, inner_channels // 4, 3, padding=1, bias=False),
+            nn.Upsample(scale_factor=2, mode='nearest'))
+        self.out2 = nn.Conv2d(
+            inner_channels, inner_channels // 4, 3, padding=1, bias=False)
+        self.inner_vit = inner_vit
+        if inner_vit:
+            # mini vit
+            self.num_keep1 = (self.img_size[0] // 64)**2
+            self.gumble_sample1 = GumbelSample(
+                inner_channels, num_keep=self.num_keep1)
+            self.pos_emb1 = nn.Parameter(
+                torch.randn(inner_channels, self.img_size[0] // 32,
+                            self.img_size[1] // 32))
+            trunc_normal_(self.pos_emb1, std=.02)
+            self.mini_vit = nn.Sequential(*[
+                ResidualAttentionBlock(
+                    inner_channels, 4, expand_ratio=2, init_values=0.1)
+                for _ in range(trans_layers)
+            ])
+        self.dropout_pos = nn.Dropout(0.1)
+        if out_sampling:
+            # sample for co-attention
+            self.num_keep2 = (self.img_size[0] // 64)**2
+            self.gumble_sample2 = GumbelSample(
+                inner_channels, num_keep=self.num_keep2)
+            self.pos_emb2 = nn.Parameter(
+                torch.randn(inner_channels, self.img_size[0] // 4,
+                            self.img_size[1] // 4))
+            trunc_normal_(self.pos_emb2, std=.02)
+        self.out_sampling = out_sampling
+
+        self.drop_path = DropPath(0.1)
+
+    def forward(self, x):
+        ms_features = self.cnn(x)
+        c2, c3, c4, c5 = ms_features
+        in5 = self.in5(c5)
+        in4 = self.in4(c4)
+        in3 = self.in3(c3)
+        in2 = self.in2(c2)
+        N, D5, H5, W5 = in5.size()
+        if self.inner_vit:
+            # random sample
+            keep_score = None
+            in5_pos = self.dropout_pos(in5 + self.pos_emb1.unsqueeze(0))
+            in5_pos_in, topk_mask = self.gumble_sample1.random_sample(in5_pos)
+            in5_pos_in = in5_pos_in.transpose(0, 1)  # [num_keep1, N, D5]
+            in5_pos_out = self.mini_vit(in5_pos_in).permute(
+                1, 2, 0)  # [N, D5, num_keep1]
+            in5 = self.gumble_sample1.restore(in5, topk_mask,
+                                              in5_pos_out.transpose(1, 2))
+        else:
+            keep_score = None
+        # FPN for fused multi-scale visual feature
+        out4 = self.up5(in5) + self.drop_path(in4)
+        out3 = self.up4(out4) + self.drop_path(in3)
+        out2 = self.up3(out3) + self.drop_path(in2)
+        p5 = self.out5(in5)
+        p4 = self.out4(out4)
+        p3 = self.out3(out3)
+        p2 = self.out2(out2)
+        feat_ms = torch.cat((p5, p4, p3, p2), 1)
+        ret_dict = dict(
+            feat_ms=feat_ms,
+            keep_score=keep_score,
+        )
+        if self.out_sampling:
+            # gumbel sampling
+            topk_mask2, gumbel_hard2, keep_score2 = self.gumble_sample2(
+                feat_ms)
+            feat_ms_pos = self.dropout_pos(feat_ms
+                                           + self.pos_emb2.unsqueeze(0))
+            feat_ms_pos_sampled = self.gumble_sample2.sample(
+                feat_ms_pos, topk_mask2,
+                gumbel_hard2).transpose(0, 1)  # [num_keep2, N, inner_c]
+            ret_dict_sup = dict(
+                feat_ms_pos_sampled=feat_ms_pos_sampled,
+                sampler=self.gumble_sample2,
+                keep_score2=keep_score2,
+            )
+            ret_dict.update(ret_dict_sup)
+        return ret_dict

modelscope/models/multi_modal/vldoc/convnext.py

@@ -0,0 +1,168 @@
+# The implementation is borrowed and partly modified from ConvNext,
+# made publicly available under the MIT License at https://github.com/facebookresearch/ConvNeXt.
+
+import os
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from timm.models.layers import DropPath, trunc_normal_
+from timm.models.registry import register_model
+
+
+class Block(nn.Module):
+    r""" ConvNeXt Block. There are two equivalent implementations:
+    (1) DwConv -> LayerNorm (channels_first) -> 1x1 Conv -> GELU -> 1x1 Conv; all in (N, C, H, W)
+    (2) DwConv -> Permute to (N, H, W, C); LayerNorm (channels_last) -> Linear -> GELU -> Linear; Permute back
+    We use (2) as we find it slightly faster in PyTorch
+
+    Args:
+        dim (int): Number of input channels.
+        drop_path (float): Stochastic depth rate. Default: 0.0
+        layer_scale_init_value (float): Init value for Layer Scale. Default: 1e-6.
+    """
+
+    def __init__(self, dim, drop_path=0., layer_scale_init_value=1e-6):
+        super().__init__()
+        self.dwconv = nn.Conv2d(
+            dim, dim, kernel_size=7, padding=3, groups=dim)  # depthwise conv
+        self.norm = LayerNorm(dim, eps=1e-6)
+        self.pwconv1 = nn.Linear(
+            dim,
+            4 * dim)  # pointwise/1x1 convs, implemented with linear layers
+        self.act = nn.GELU()
+        self.pwconv2 = nn.Linear(4 * dim, dim)
+        self.gamma = nn.Parameter(
+            layer_scale_init_value * torch.ones((dim)),
+            requires_grad=True) if layer_scale_init_value > 0 else None
+        self.drop_path = DropPath(
+            drop_path) if drop_path > 0. else nn.Identity()
+
+    def forward(self, x):
+        input = x
+        x = self.dwconv(x)
+        x = x.permute(0, 2, 3, 1)  # (N, C, H, W) -> (N, H, W, C)
+        x = self.norm(x)
+        x = self.pwconv1(x)
+        x = self.act(x)
+        x = self.pwconv2(x)
+        if self.gamma is not None:
+            x = self.gamma * x
+        x = x.permute(0, 3, 1, 2)  # (N, H, W, C) -> (N, C, H, W)
+
+        x = input + self.drop_path(x)
+        return x
+
+
+class ConvNeXt(nn.Module):
+    r""" ConvNeXt
+        A PyTorch impl of : `A ConvNet for the 2020s`  -
+          https://arxiv.org/pdf/2201.03545.pdf
+    Args:
+        in_chans (int): Number of input image channels. Default: 3
+        num_classes (int): Number of classes for classification head. Default: 1000
+        depths (tuple(int)): Number of blocks at each stage. Default: [3, 3, 9, 3]
+        dims (int): Feature dimension at each stage. Default: [96, 192, 384, 768]
+        drop_path_rate (float): Stochastic depth rate. Default: 0.
+        layer_scale_init_value (float): Init value for Layer Scale. Default: 1e-6.
+        head_init_scale (float): Init scaling value for classifier weights and biases. Default: 1.
+    """
+
+    def __init__(
+        self,
+        in_chans=3,
+        depths=[3, 3, 9, 3],
+        dims=[96, 192, 384, 768],
+        drop_path_rate=0.,
+        layer_scale_init_value=1e-6,
+    ):
+        super().__init__()
+
+        self.downsample_layers = nn.ModuleList(
+        )  # stem and 3 intermediate downsampling conv layers
+        stem = nn.Sequential(
+            nn.Conv2d(in_chans, dims[0], kernel_size=4, stride=4),
+            LayerNorm(dims[0], eps=1e-6, data_format='channels_first'))
+        self.downsample_layers.append(stem)
+        for i in range(3):
+            downsample_layer = nn.Sequential(
+                LayerNorm(dims[i], eps=1e-6, data_format='channels_first'),
+                nn.Conv2d(dims[i], dims[i + 1], kernel_size=2, stride=2),
+            )
+            self.downsample_layers.append(downsample_layer)
+
+        self.stages = nn.ModuleList(
+        )  # 4 feature resolution stages, each consisting of multiple residual blocks
+        dp_rates = [
+            x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))
+        ]
+        cur = 0
+        for i in range(4):
+            stage = nn.Sequential(*[
+                Block(
+                    dim=dims[i],
+                    drop_path=dp_rates[cur + j],
+                    layer_scale_init_value=layer_scale_init_value)
+                for j in range(depths[i])
+            ])
+            self.stages.append(stage)
+            cur += depths[i]
+
+        # self.norm = nn.LayerNorm(dims[-1], eps=1e-6) # final norm layer
+        self.dims = dims
+
+        self.apply(self._init_weights)
+
+    def _init_weights(self, m):
+        if isinstance(m, (nn.Conv2d, nn.Linear)):
+            trunc_normal_(m.weight, std=.02)
+            nn.init.constant_(m.bias, 0)
+
+    def forward(self, x):
+        xs = []
+        for i in range(4):
+            x = self.downsample_layers[i](x)
+            x = self.stages[i](x)
+            xs.append(x)
+        # x = x.permute(0, 2, 3, 1) # [N, H, W, C]
+        # x = self.norm(x)
+        # x = x.permute(0, 3, 1, 2) # [N, C, H, W]
+        return tuple(xs)
+
+
+class LayerNorm(nn.Module):
+    r""" LayerNorm that supports two data formats: channels_last (default) or channels_first.
+    The ordering of the dimensions in the inputs. channels_last corresponds to inputs with
+    shape (batch_size, height, width, channels) while channels_first corresponds to inputs
+    with shape (batch_size, channels, height, width).
+    """
+
+    def __init__(self,
+                 normalized_shape,
+                 eps=1e-6,
+                 data_format='channels_last'):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(normalized_shape))
+        self.bias = nn.Parameter(torch.zeros(normalized_shape))
+        self.eps = eps
+        self.data_format = data_format
+        if self.data_format not in ['channels_last', 'channels_first']:
+            raise NotImplementedError
+        self.normalized_shape = (normalized_shape, )
+
+    def forward(self, x):
+        if self.data_format == 'channels_last':
+            return F.layer_norm(x, self.normalized_shape, self.weight,
+                                self.bias, self.eps)
+        elif self.data_format == 'channels_first':
+            u = x.mean(1, keepdim=True)
+            s = (x - u).pow(2).mean(1, keepdim=True)
+            x = (x - u) / torch.sqrt(s + self.eps)
+            x = self.weight[:, None, None] * x + self.bias[:, None, None]
+            return x
+
+
+@register_model
+def convnext_tiny(pretrained=False, in_22k=False, **kwargs):
+    model = ConvNeXt(depths=[3, 3, 9, 3], dims=[96, 192, 384, 768], **kwargs)
+    return model

modelscope/models/multi_modal/vldoc/model.py

@@ -0,0 +1,433 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import copy
+import logging
+import math
+import os
+import re
+import sys
+
+import json
+import torch
+import torch.distributed as dist
+import torch.nn as nn
+from torchvision.ops import roi_align
+
+from modelscope.metainfo import Models
+from modelscope.models import TorchModel
+from modelscope.models.builder import MODELS
+from modelscope.models.multi_modal.vldoc.conv_fpn_trans import FPNTrans
+from modelscope.models.multi_modal.vldoc.modeling_layout_roberta import (
+    LayoutRobertaModel, LayoutRobertaPreTrainedModel)
+from modelscope.models.multi_modal.vldoc.transformer_local import (
+    TransformerDecoder, TransformerDecoderLayer)
+from modelscope.utils.constant import ModeKeys, ModelFile, Tasks
+from modelscope.utils.logger import get_logger
+
+logger = get_logger()
+
+__all__ = ['VLDocForDocVLEmbedding']
+
+
+class GeoVLDocModelOutputs(object):
+
+    def __init__(
+        self,
+        text_features,
+        text_mm_features,
+        block_vis_features,
+        block_vis_mm_features,
+        image_mm_features,
+    ):
+        # [batch size, sequence length, hidden size]
+        self.text_features = text_features
+        # [batch size, sequence length, hidden size]
+        self.text_mm_features = text_mm_features
+        # [batch size, block num, hidden size]
+        self.block_vis_features = block_vis_features
+        # [batch size, block num, hidden size]
+        self.block_vis_mm_features = block_vis_mm_features
+        # [batch size, hidden size]
+        self.image_mm_features = image_mm_features
+
+
+class GeoVLDocModel(LayoutRobertaPreTrainedModel):
+
+    def __init__(self, config, hard_negtive_sampling=False):
+        super().__init__(config)
+        self.config = config
+        self.hard_negtive_sampling = hard_negtive_sampling
+
+        if getattr(self.config, 'architectures', None):
+            if self.config.architectures[0] == 'LayoutRobertaModel':
+                self.text_encoder = LayoutRobertaModel(config)
+            else:
+                self.text_encoder = LayoutRobertaModel(config)
+        else:
+            self.text_encoder = LayoutRobertaModel(config)
+        self.visual_encoder = FPNTrans(
+            img_size=self.config.image_size, inner_vit=False)
+        self.pool = nn.AdaptiveAvgPool2d([1, 1])
+        self.vis_linear = nn.Linear(256, self.config.hidden_size)
+
+        cross_modal_text_layer = TransformerDecoderLayer(
+            self.config.hidden_size,
+            self.config.num_attention_heads,
+            self.config.intermediate_size,
+            self_attn=True)
+        self.cross_modal_text = TransformerDecoder(cross_modal_text_layer, 1)
+
+        cross_modal_visual_layer = TransformerDecoderLayer(
+            self.config.hidden_size,
+            self.config.num_attention_heads,
+            self.config.intermediate_size,
+            self_attn=True)
+        self.cross_modal_visual = TransformerDecoder(cross_modal_visual_layer,
+                                                     1)
+
+        self.init_weights()
+
+    def from_pretrained(self, ckpt_path: str):
+        state_dict = torch.load(ckpt_path, map_location='cpu')
+        state_dict_new = {}
+        for k, v in state_dict.items():
+            k = k.replace('geo_vl_doc_model.', '')
+            state_dict_new[k] = v
+        self.load_state_dict(state_dict_new)
+
+    def forward(self,
+                input_ids=None,
+                image=None,
+                bbox=None,
+                bbox_4p_normalized=None,
+                attention_mask=None,
+                first_token_idxes=None,
+                first_token_idxes_mask=None,
+                token_type_ids=None,
+                position_ids=None,
+                head_mask=None,
+                inputs_embeds=None,
+                encoder_hidden_states=None,
+                encoder_attention_mask=None,
+                past_key_values=None,
+                use_cache=None,
+                output_attentions=None,
+                output_hidden_states=None,
+                return_dict=None,
+                **kwargs):
+
+        batch_size, seq_len = input_ids.shape
+
+        return_dict = (
+            return_dict
+            if return_dict is not None else self.config.use_return_dict)
+
+        kwargs['line_bbox'] = bbox
+        # ################ get text representation ################
+        if self.config.architectures[0] == 'LayoutRobertaModel':
+            outputs = self.text_encoder(
+                input_ids,
+                bbox=bbox_4p_normalized,
+                attention_mask=attention_mask,
+                token_type_ids=token_type_ids,
+                position_ids=position_ids,
+                head_mask=head_mask,
+                inputs_embeds=inputs_embeds,
+                output_attentions=output_attentions,
+                output_hidden_states=output_hidden_states,
+                return_dict=return_dict,
+                **kwargs)
+        else:
+            outputs = self.text_encoder(
+                input_ids,
+                bbox=bbox_4p_normalized,
+                attention_mask=attention_mask,
+                token_type_ids=token_type_ids,
+                position_ids=position_ids,
+                head_mask=head_mask,
+                inputs_embeds=inputs_embeds,
+                output_attentions=output_attentions,
+                output_hidden_states=output_hidden_states,
+                return_dict=return_dict,
+                **kwargs)
+
+        # sequence_output: [batch_size, seq_len, hidden_size]
+        # pooled_output: [batch_size, hidden_size]
+        sequence_output, pooled_output = outputs[:2]
+
+        # ################ get visual representation ################
+        _, num_first = first_token_idxes.shape
+        B_batch_dim = torch.arange(
+            0, batch_size,
+            device=input_ids.device).reshape(batch_size,
+                                             1).expand(batch_size, num_first)
+
+        feature_bbox = bbox[B_batch_dim, first_token_idxes]
+        _, block_num, _ = feature_bbox.shape
+
+        visual_out = self.visual_encoder(image)
+        batch_idxs = torch.arange(
+            0, batch_size, device=sequence_output.device).reshape(
+                batch_size, 1).expand(batch_size, block_num).unsqueeze(-1)
+
+        # [batch_size*block_num, 5]
+        batch_idx_with_bbox = torch.cat(
+            (batch_idxs, feature_bbox),
+            2).reshape(batch_size * block_num,
+                       5).to(dtype=visual_out['feat_ms'].dtype)
+
+        if visual_out['feat_ms'].dtype == torch.float16:
+            # [batch_size*block_num, 256, 1, 1]
+            blk_vis_features = roi_align(
+                visual_out['feat_ms'].to(torch.float32),
+                batch_idx_with_bbox.to(torch.float32),
+                1,
+                spatial_scale=visual_out['feat_ms'].size(-1) / 1000.0)
+            blk_vis_features = blk_vis_features.to(
+                dtype=visual_out['feat_ms'].dtype)
+        else:
+            blk_vis_features = roi_align(
+                visual_out['feat_ms'],
+                batch_idx_with_bbox.to(torch.float32),
+                1,
+                spatial_scale=visual_out['feat_ms'].size(-1) / 1000.0)
+
+        # [batch_size*block_num, 256]
+        blk_vis_features = blk_vis_features.squeeze(2).squeeze(2).reshape(
+            batch_size, block_num, 256)
+
+        # visual block features:
+        # blk_vis_features: [batch_size, block_num, hidden_size]
+        blk_vis_features = self.vis_linear(blk_vis_features)
+        blk_vis_features = blk_vis_features * first_token_idxes_mask.unsqueeze(
+            2)
+        # [batch_size, 256]
+        full_img_features = self.pool(
+            visual_out['feat_ms']).squeeze(2).squeeze(2)
+        # [batch_size, hidden_size]
+        full_img_features = self.vis_linear(full_img_features).unsqueeze(1)
+
+        # ################ multi-modal fusion ################
+
+        # cross attention inputs
+        vis_inps = torch.cat((full_img_features, blk_vis_features), 1)
+
+        glb_feat_attn = torch.ones((batch_size, 1)).to(input_ids.device)
+
+        vis_mask = torch.cat((glb_feat_attn, first_token_idxes_mask), 1)
+
+        # When we use transformer in torch.nn, the input size is
+        # [seq_len, batch_size, hidden_size]
+        # In attention_mask, 1 denotes masked
+        new_attention_mask = (1 - attention_mask) > 0
+        new_vis_mask = (1 - vis_mask) > 0
+
+        text_mm_feat = self.cross_modal_text(
+            tgt=sequence_output.transpose(0, 1),
+            memory=vis_inps.transpose(0, 1),
+            tgt_key_padding_mask=new_attention_mask,
+            memory_key_padding_mask=new_vis_mask)
+
+        vis_mm_feat = self.cross_modal_visual(
+            tgt=vis_inps.transpose(0, 1),
+            memory=sequence_output.transpose(0, 1),
+            tgt_key_padding_mask=new_vis_mask,
+            memory_key_padding_mask=new_attention_mask,
+        )
+
+        # [batch_size, seq_len, hidden_size]
+        text_mm_feat = text_mm_feat.transpose(0, 1)
+        # [batch_size, 1+block_num, hidden_size]
+        vis_mm_feat = vis_mm_feat.transpose(0, 1)
+
+        # image_mm_features = vis_mm_feat[:, 0, :]
+        block_vis_mm_features = vis_mm_feat[:, 1:]
+
+        return GeoVLDocModelOutputs(
+            text_features=sequence_output,
+            text_mm_features=text_mm_feat,
+            block_vis_features=blk_vis_features,
+            block_vis_mm_features=block_vis_mm_features,
+            image_mm_features=vis_mm_feat,
+        )
+
+
+@MODELS.register_module(Tasks.document_vl_embedding, module_name=Models.vldoc)
+class VLDocForDocVLEmbedding(TorchModel):
+    """
+    Generate multi-modal document embeddings in segment-level and token-level.
+
+    Args:
+        model_dir:
+            the path in model hub, e.g., 'damo/multi-modal_convnext-roberta-base_vldoc-embedding'
+    """
+
+    def __init__(self, model_dir: str, *args, **kwargs):
+        super().__init__(model_dir=model_dir, *args, **kwargs)
+
+        # Initialize the model.
+        from modelscope.models.multi_modal.vldoc.modeling_layout_roberta import LayoutRobertaConfig
+        model_cfg_path = os.path.join(model_dir, 'config.json')
+        logger.info('Loading config file from {}'.format(model_cfg_path))
+        assert os.path.exists(model_cfg_path)
+        self.config = LayoutRobertaConfig.from_json_file(model_cfg_path)
+        self.doc_model = GeoVLDocModel(self.config)
+
+        # restore the pretrained weight
+        model_path = os.path.join(model_dir, ModelFile.TORCH_MODEL_FILE)
+        assert os.path.exists(model_path)
+        self.doc_model.from_pretrained(model_path)
+        logger.info('Loading model from {}'.format(model_path))
+
+        # Initialize the tokenizer.
+        from modelscope.models.multi_modal.vldoc.tokenization import VLDocXLMTokenizer
+        tokenizer_path = os.path.join(model_dir, ModelFile.TOKENIZER_FOLDER)
+        self.tokenizer = VLDocXLMTokenizer.from_pretrained(tokenizer_path)
+
+        # place the model
+        self.device = 'cuda:{}'.format(int(os.environ.get(
+            'LOCAL_RANK', 0))) if torch.cuda.is_available() else 'cpu'
+        if torch.cuda.is_available():
+            self.doc_model.to(self.device)
+            logger.info('Use GPU {} for finetuning & inference'.format(
+                int(os.environ.get('LOCAL_RANK', 0))))
+        else:
+            self.doc_model.float()
+            logger.info('Use CPU for finetuning & inference')
+
+    def forward(self,
+                input_ids=None,
+                image=None,
+                bbox=None,
+                bbox_4p_normalized=None,
+                attention_mask=None,
+                first_token_idxes=None,
+                first_token_idxes_mask=None,
+                token_type_ids=None,
+                position_ids=None,
+                head_mask=None,
+                inputs_embeds=None,
+                encoder_hidden_states=None,
+                encoder_attention_mask=None,
+                past_key_values=None,
+                use_cache=None,
+                output_attentions=None,
+                output_hidden_states=None,
+                return_dict=None,
+                **kwargs):
+        """
+        Args:
+            - input_ids: :math:`(B, T, E)`, the input tokens, where B is the batch size,
+              T is the max token size, E is the embedding dimension.
+            - image: :math:`(B, C, H, W)`, normalized images.
+            - bbox: :math:`(B, T, 4)`, segment boxes denoted by top-left and bottom-right
+              vertexes whose values are normalized to [0, 1000).
+            - bbox_4p_normalized: :math:`(B, T, 8)`, word boxes denoted by 4 vertexes, whose
+              values are normalized to [0, 1).
+            - attention_mask: :math:`(B, T)`, mask for input tokens, where 0 means masked.
+            - first_token_idxes: :math:`(B, S)`, indexes of the corresponding first tokens
+              of all segments, where S is the max segment size.
+            - first_token_idxes_mask: :math:`(B, S)`, mask for segments, where 0 means masked.
+        Optional:
+            - line_rank_id: :math:`(B, T)`, orders of segments.
+            - line_rank_inner_id: :math:`(B, T)`, BIE-like tags.
+
+        To be more specific, please refer to the class `TextLayoutSerializer` in
+          `modelscope/models/multi_modal/vldoc/processing.py`.
+        """
+
+        vldoc_outputs = self.doc_model(
+            input_ids=input_ids,
+            image=image,
+            bbox=bbox,
+            bbox_4p_normalized=bbox_4p_normalized,
+            attention_mask=attention_mask,
+            first_token_idxes=first_token_idxes,
+            first_token_idxes_mask=first_token_idxes_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            encoder_hidden_states=encoder_hidden_states,
+            encoder_attention_mask=encoder_attention_mask,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            **kwargs)
+
+        return dict(
+            img_embedding=vldoc_outputs.image_mm_features,
+            text_embedding=vldoc_outputs.text_mm_features,
+        )
+
+
+def init_pretrained_weight(
+    model,
+    pretrained_model_path,
+    state_dict=None,
+    cache_dir=None,
+    init_backbone='roberta',
+):
+    if state_dict is None:
+        state_dict = torch.load(pretrained_model_path, map_location='cpu')
+
+    old_keys = []
+    new_keys = []
+    state_dict_keys = list(state_dict.keys())
+
+    if init_backbone == 'roberta':
+        for i in range(len(state_dict_keys)):
+            key = state_dict_keys[i]
+            new_key = None
+
+            if key.startswith('roberta.'):
+                new_key = key.replace('roberta.',
+                                      'geo_vl_doc_model.text_encoder.')
+                key = copy.deepcopy(new_key)
+
+            if new_key:
+                old_keys.append(state_dict_keys[i])
+                new_keys.append(new_key)
+
+    for old_key, new_key in zip(old_keys, new_keys):
+        state_dict[new_key] = state_dict.pop(old_key)
+
+    missing_keys = []
+    unexpected_keys = []
+    error_msgs = []
+
+    # copy state_dict so _load_from_state_dict can modify it
+    metadata = getattr(state_dict, '_metadata', None)
+    state_dict = state_dict.copy()
+    if metadata is not None:
+        state_dict._metadata = metadata
+
+    def load(module, prefix=''):
+        local_metadata = {} if metadata is None else metadata.get(
+            prefix[:-1], {})
+        module._load_from_state_dict(state_dict, prefix, local_metadata, True,
+                                     missing_keys, unexpected_keys, error_msgs)
+        for name, child in module._modules.items():
+            if child is not None:
+                load(child, prefix + name + '.')
+
+    start_prefix = ''
+    if not hasattr(model, 'geo_vl_doc_model') and any(
+            s.startswith('geo_vl_doc_model.') for s in state_dict.keys()):
+        start_prefix = 'geo_vl_doc_model.'
+    load(model, prefix=start_prefix)
+    if len(missing_keys) > 0:
+        logger.info(
+            'Weights of {} not initialized from pretrained model: {}'.format(
+                model.__class__.__name__, missing_keys))
+    if len(unexpected_keys) > 0:
+        logger.info('Weights from pretrained model not used in {}: {}'.format(
+            model.__class__.__name__, unexpected_keys))
+    if len(error_msgs) > 0:
+        raise RuntimeError(
+            'Error(s) in loading state_dict for {}:\n\t{}'.format(
+                model.__class__.__name__, '\n\t'.join(error_msgs)))
+
+    return model

modelscope/models/multi_modal/vldoc/processing.py

@@ -0,0 +1,538 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+"""
+Processor class for GeoLayoutLM.
+"""
+
+from collections import defaultdict
+from typing import Dict, Iterable, List, Union
+
+import cv2
+import numpy as np
+import PIL
+import torch
+from timm.data.constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD
+from torchvision import transforms
+
+from modelscope.preprocessors.image import LoadImage
+
+
+def custom_tokenize(tokenizer, text):
+    toks = tokenizer.tokenize('pad ' + text)[1:]
+    toks2 = toks[1:] if len(toks) > 0 and toks[0] == '▁' else toks
+    return toks2
+
+
+class ImageProcessor(object):
+    r"""
+    Construct a GeoLayoutLM image processor
+    Args:
+        do_preprocess (`bool`): whether to do preprocess to unify the image format,
+            resize and convert to tensor.
+        do_rescale: only works when we disable do_preprocess.
+    """
+
+    def __init__(self,
+                 do_preprocess: bool = True,
+                 do_resize: bool = False,
+                 image_size: Dict[str, int] = None,
+                 do_rescale: bool = False,
+                 rescale_factor: float = 1. / 255,
+                 do_normalize: bool = True,
+                 image_mean: Union[float, Iterable[float]] = None,
+                 image_std: Union[float, Iterable[float]] = None,
+                 apply_ocr: bool = True,
+                 **kwargs) -> None:
+        self.do_preprocess = do_preprocess
+        self.do_resize = do_resize
+        self.size = image_size if image_size is not None else {
+            'height': 768,
+            'width': 768
+        }
+        self.do_rescale = do_rescale and (not do_preprocess)
+        self.rescale_factor = rescale_factor
+        self.do_normalize = do_normalize
+        image_mean = IMAGENET_DEFAULT_MEAN if image_mean is None else image_mean
+        image_std = IMAGENET_DEFAULT_STD if image_std is None else image_std
+        self.image_mean = (image_mean, image_mean, image_mean) if isinstance(
+            image_mean, float) else image_mean
+        self.image_std = (image_std, image_std, image_std) if isinstance(
+            image_std, float) else image_std
+        self.apply_ocr = apply_ocr
+        self.kwargs = kwargs
+
+        self.totensor = transforms.ToTensor()
+
+    def preprocess(self, image: Union[np.ndarray, PIL.Image.Image]):
+        """ unify the image format, resize and convert to tensor.
+        """
+        image = LoadImage.convert_to_ndarray(image)[:, :, ::-1]
+        size_raw = image.shape[:2]
+        if self.do_resize:
+            image = cv2.resize(image,
+                               (self.size['width'], self.size['height']))
+        # convert to pytorch tensor
+        image_pt = self.totensor(image)
+        return image_pt, size_raw
+
+    def __call__(self, images: Union[list, np.ndarray, PIL.Image.Image, str]):
+        """
+        Args:
+            images: list of np.ndarrays, PIL images or image tensors.
+        """
+        if not isinstance(images, list):
+            images = [images]
+        sizes_raw = []
+        if self.do_preprocess:
+            for i in range(len(images)):
+                images[i], size_raw = self.preprocess(images[i])
+                sizes_raw.append(size_raw)
+        images_pt = torch.stack(images, dim=0)  # [b, c, h, w]
+        if self.do_rescale:
+            images_pt = images_pt * self.rescale_factor
+        if self.do_normalize:
+            mu = torch.tensor(self.image_mean).view(1, 3, 1, 1)
+            std = torch.tensor(self.image_std).view(1, 3, 1, 1)
+            images_pt = (images_pt - mu) / (std + 1e-8)
+
+        # TODO: apply OCR
+        ocr_infos = None
+        if self.apply_ocr:
+            raise NotImplementedError('OCR service is not available yet!')
+        if len(sizes_raw) == 0:
+            sizes_raw = None
+        data = {
+            'images': images_pt,
+            'ocr_infos': ocr_infos,
+            'sizes_raw': sizes_raw
+        }
+        return data
+
+
+class OCRUtils(object):
+
+    def __init__(self):
+        self.version = 'v0'
+
+    def __call__(self, ocr_infos):
+        """
+        sort boxes, filtering or other preprocesses
+        should return sorted ocr_infos
+        """
+        raise NotImplementedError
+
+
+def bound_box(box, height, width):
+    # box: [x_tl, y_tl, x_br, y_br] or ...
+    assert len(box) == 4 or len(box) == 8
+    for i in range(len(box)):
+        if i & 1:
+            box[i] = max(0, min(box[i], height))
+        else:
+            box[i] = max(0, min(box[i], width))
+    return box
+
+
+def bbox2pto4p(box2p):
+    box4p = [
+        box2p[0], box2p[1], box2p[2], box2p[1], box2p[2], box2p[3], box2p[0],
+        box2p[3]
+    ]
+    return box4p
+
+
+def bbox4pto2p(box4p):
+    box2p = [
+        min(box4p[0], box4p[2], box4p[4], box4p[6]),
+        min(box4p[1], box4p[3], box4p[5], box4p[7]),
+        max(box4p[0], box4p[2], box4p[4], box4p[6]),
+        max(box4p[1], box4p[3], box4p[5], box4p[7]),
+    ]
+    return box2p
+
+
+def stack_tensor_dict(tensor_dicts: List[Dict[str, torch.Tensor]]):
+    one_dict = defaultdict(list)
+    for td in tensor_dicts:
+        for k, v in td.items():
+            one_dict[k].append(v)
+    res_dict = {}
+    for k, v in one_dict.items():
+        res_dict[k] = torch.stack(v, dim=0)
+    return res_dict
+
+
+class TextLayoutSerializer(object):
+
+    def __init__(self,
+                 max_seq_length: int,
+                 max_block_num: int,
+                 tokenizer,
+                 width=768,
+                 height=768,
+                 use_roberta_tokenizer: bool = True,
+                 ocr_utils: OCRUtils = None):
+        self.version = 'v0'
+
+        self.max_seq_length = max_seq_length
+        self.max_block_num = max_block_num
+        self.tokenizer = tokenizer
+        self.width = width
+        self.height = height
+        self.use_roberta_tokenizer = use_roberta_tokenizer
+        self.ocr_utils = ocr_utils
+
+        self.pad_token_id = tokenizer.pad_token_id
+        self.cls_token_id = tokenizer.bos_token_id
+        self.sep_token_id = tokenizer.eos_token_id
+        self.unk_token_id = tokenizer.unk_token_id
+        self.cls_bbs_word = [0.0] * 8
+        self.cls_bbs_line = [0] * 4
+
+    def label2seq(self, ocr_info: list, label_info: list):
+        raise NotImplementedError
+
+    def serialize_single(
+        self,
+        ocr_info: list = None,
+        input_ids: list = None,
+        bbox_line: List[List] = None,
+        bbox_word: List[List] = None,
+        width: int = 768,
+        height: int = 768,
+    ):
+        r"""
+        Either ocr_info or (input_ids, bbox_line, bbox_word)
+            should be provided.
+        If (input_ids, bbox_line, bbox_word) is provided,
+            convinient plug into the serialization (customization)
+            is offered. The tokens must be organised by blocks and words.
+        Else, ocr_info must be provided, to be parsed
+            to sequences directly (the simplest way).
+        Args:
+            ocr_info: [
+                {"text": "xx", "box": [a,b,c,d],
+                 "words": [{"text": "x", "box": [e,f,g,h]}, ...]},
+                ...
+            ]
+            bbox_line: the coordinate value should match the original image
+                (i.e., not be normalized).
+        """
+        if input_ids is not None:
+            assert len(input_ids) == len(bbox_line)
+            assert len(input_ids) == len(bbox_word)
+            input_ids, bbs_word, bbs_line, first_token_idxes, \
+                line_rank_ids, line_rank_inner_ids, word_rank_ids = \
+                self.halfseq2seq(input_ids, bbox_line, bbox_word, width, height)
+        else:
+            assert ocr_info is not None
+            input_ids, bbs_word, bbs_line, first_token_idxes, \
+                line_rank_ids, line_rank_inner_ids, word_rank_ids = \
+                self.ocr_info2seq(ocr_info, width, height)
+
+        token_seq = {}
+        token_seq['input_ids'] = torch.ones(
+            self.max_seq_length, dtype=torch.int64) * self.pad_token_id
+        token_seq['attention_mask'] = torch.zeros(
+            self.max_seq_length, dtype=torch.int64)
+        token_seq['first_token_idxes'] = torch.zeros(
+            self.max_block_num, dtype=torch.int64)
+        token_seq['first_token_idxes_mask'] = torch.zeros(
+            self.max_block_num, dtype=torch.int64)
+        token_seq['bbox_4p_normalized'] = torch.zeros(
+            self.max_seq_length, 8, dtype=torch.float32)
+        token_seq['bbox'] = torch.zeros(
+            self.max_seq_length, 4, dtype=torch.float32)
+        token_seq['line_rank_id'] = torch.zeros(
+            self.max_seq_length, dtype=torch.int64)  # start from 1
+        token_seq['line_rank_inner_id'] = torch.ones(
+            self.max_seq_length, dtype=torch.int64)  # 1 2 2 3
+        token_seq['word_rank_id'] = torch.zeros(
+            self.max_seq_length, dtype=torch.int64)  # start from 1
+
+        # expand using cls and sep tokens
+        sep_bbs_word = [width, height] * 4
+        sep_bbs_line = [width, height] * 2
+        input_ids = [self.cls_token_id] + input_ids + [self.sep_token_id]
+        bbs_line = [self.cls_bbs_line] + bbs_line + [sep_bbs_line]
+        bbs_word = [self.cls_bbs_word] + bbs_word + [sep_bbs_word]
+
+        # assign
+        len_tokens = len(input_ids)
+        len_lines = len(first_token_idxes)
+        token_seq['input_ids'][:len_tokens] = torch.tensor(input_ids)
+        token_seq['attention_mask'][:len_tokens] = 1
+        token_seq['first_token_idxes'][:len_lines] = torch.tensor(
+            first_token_idxes)
+        token_seq['first_token_idxes_mask'][:len_lines] = 1
+        token_seq['line_rank_id'][1:len_tokens
+                                  - 1] = torch.tensor(line_rank_ids)
+        token_seq['line_rank_inner_id'][1:len_tokens - 1] = torch.tensor(
+            line_rank_inner_ids)
+        token_seq['line_rank_inner_id'] = token_seq[
+            'line_rank_inner_id'] * token_seq['attention_mask']
+        token_seq['word_rank_id'][1:len_tokens
+                                  - 1] = torch.tensor(word_rank_ids)
+
+        token_seq['bbox_4p_normalized'][:len_tokens, :] = torch.tensor(
+            bbs_word)
+        # word bbox normalization -> [0, 1]
+        token_seq['bbox_4p_normalized'][:, [0, 2, 4, 6]] = \
+            token_seq['bbox_4p_normalized'][:, [0, 2, 4, 6]] / width
+        token_seq['bbox_4p_normalized'][:, [1, 3, 5, 7]] = \
+            token_seq['bbox_4p_normalized'][:, [1, 3, 5, 7]] / height
+
+        token_seq['bbox'][:len_tokens, :] = torch.tensor(bbs_line)
+        # line bbox -> [0, 1000)
+        token_seq['bbox'][:,
+                          [0, 2]] = token_seq['bbox'][:, [0, 2]] / width * 1000
+        token_seq['bbox'][:,
+                          [1, 3]] = token_seq['bbox'][:,
+                                                      [1, 3]] / height * 1000
+        token_seq['bbox'] = token_seq['bbox'].long()
+
+        return token_seq
+
+    def ocr_info2seq(self, ocr_info: list, width: int, height: int):
+        input_ids = []
+        bbs_word = []
+        bbs_line = []
+        first_token_idxes = []
+        line_rank_ids = []
+        line_rank_inner_ids = []
+        word_rank_ids = []
+
+        early_stop = False
+        for line_idx, line in enumerate(ocr_info):
+            if line_idx == self.max_block_num:
+                early_stop = True
+            if early_stop:
+                break
+            lbox = line['box']
+            lbox = bound_box(lbox, height, width)
+            is_first_word = True
+            for word_id, word_info in enumerate(line['words']):
+                wtext = word_info['text']
+                wbox = word_info['box']
+                wbox = bound_box(wbox, height, width)
+                wbox4p = bbox2pto4p(wbox)
+                if self.use_roberta_tokenizer:
+                    wtokens = custom_tokenize(self.tokenizer, wtext)
+                else:
+                    wtokens = self.tokenizer.tokenize(wtext)
+                wtoken_ids = self.tokenizer.convert_tokens_to_ids(wtokens)
+                if len(wtoken_ids) == 0:
+                    wtoken_ids.append(self.unk_token_id)
+                n_tokens = len(wtoken_ids)
+                # reserve for cls and sep
+                if len(input_ids) + n_tokens > self.max_seq_length - 2:
+                    early_stop = True
+                    break  # chunking early for long documents
+                if is_first_word:
+                    first_token_idxes.append(len(input_ids) + 1)
+                input_ids.extend(wtoken_ids)
+                bbs_word.extend([wbox4p] * n_tokens)
+                bbs_line.extend([lbox] * n_tokens)
+                word_rank_ids.extend([word_id + 1] * n_tokens)
+                line_rank_ids.extend([line_idx + 1] * n_tokens)
+                if is_first_word:
+                    if len(line_rank_inner_ids
+                           ) > 0 and line_rank_inner_ids[-1] == 2:
+                        line_rank_inner_ids[-1] = 3
+                    line_rank_inner_ids.extend([1] + (n_tokens - 1) * [2])
+                    is_first_word = False
+                else:
+                    line_rank_inner_ids.extend(n_tokens * [2])
+        if len(line_rank_inner_ids) > 0 and line_rank_inner_ids[-1] == 2:
+            line_rank_inner_ids[-1] = 3
+
+        return input_ids, bbs_word, bbs_line, first_token_idxes, line_rank_ids, \
+            line_rank_inner_ids, word_rank_ids
+
+    def halfseq2seq(self, input_ids: list, bbox_line: List[List],
+                    bbox_word: List[List], width: int, height: int):
+        """
+        for convinient plug into the serialization, given the 3 customized sequences.
+        They should not contain special tokens like [CLS] or [SEP].
+        """
+        bbs_word = []
+        bbs_line = []
+        first_token_idxes = []
+        line_rank_ids = []
+        line_rank_inner_ids = []
+        word_rank_ids = []
+
+        n_real_tokens = len(input_ids)
+        lb_prev, wb_prev = None, None
+        line_id = 0
+        word_id = 1
+        for i in range(n_real_tokens):
+            lb_now = bbox_line[i]
+            wb_now = bbox_word[i]
+            line_start = lb_prev is None or lb_now != lb_prev
+            word_start = wb_prev is None or wb_now != wb_prev
+            lb_prev, wb_prev = lb_now, wb_now
+
+            if len(lb_now) == 8:
+                lb_now = bbox4pto2p(lb_now)
+            assert len(lb_now) == 4
+            lb_now = bound_box(lb_now, height, width)
+            if len(wb_now) == 4:
+                wb_now = bbox2pto4p(wb_now)
+            assert len(wb_now) == 8
+            wb_now = bound_box(wb_now, height, width)
+
+            bbs_word.append(wb_now)
+            bbs_line.append(lb_now)
+
+            if word_start:
+                word_id += 1
+            if line_start:
+                line_id += 1
+                first_token_idxes.append(i + 1)
+                if len(line_rank_inner_ids
+                       ) > 0 and line_rank_inner_ids[-1] == 2:
+                    line_rank_inner_ids[-1] = 3
+                line_rank_inner_ids.append(1)
+                word_id = 1
+            else:
+                line_rank_inner_ids.append(2)
+            line_rank_ids.append(line_id)
+            word_rank_ids.append(word_id)
+
+        if len(line_rank_inner_ids) > 0 and line_rank_inner_ids[-1] == 2:
+            line_rank_inner_ids[-1] = 3
+
+        return input_ids, bbs_word, bbs_line, first_token_idxes, \
+            line_rank_ids, line_rank_inner_ids, word_rank_ids
+
+    def __call__(
+        self,
+        ocr_infos: List[List] = None,
+        input_ids: list = None,
+        bboxes_line: List[List] = None,
+        bboxes_word: List[List] = None,
+        sizes_raw: list = None,
+        **kwargs,
+    ):
+        n_samples = len(ocr_infos) if ocr_infos is not None else len(input_ids)
+        if sizes_raw is None:
+            sizes_raw = [(self.height, self.width)] * n_samples
+        seqs = []
+        if input_ids is not None:
+            assert len(input_ids) == len(bboxes_line)
+            assert len(input_ids) == len(bboxes_word)
+            for input_id, bbox_line, bbox_word, size_raw in zip(
+                    input_ids, bboxes_line, bboxes_word, sizes_raw):
+                height, width = size_raw
+                token_seq = self.serialize_single(None, input_id, bbox_line,
+                                                  bbox_word, width, height)
+                seqs.append(token_seq)
+        else:
+            assert ocr_infos is not None, 'For serialization, ocr_infos must not be NoneType!'
+            if self.ocr_utils is not None:
+                ocr_infos = self.ocr_utils(ocr_infos)
+            for ocr_info, size_raw in zip(ocr_infos, sizes_raw):
+                height, width = size_raw
+                token_seq = self.serialize_single(
+                    ocr_info, width=width, height=height)
+                seqs.append(token_seq)
+        pt_seqs = stack_tensor_dict(seqs)
+        return pt_seqs
+
+
+class Processor(object):
+    r"""Construct a GeoLayoutLM processor.
+
+    Args:
+        max_seq_length: max length for token
+        max_block_num: max number of text lines (blocks or segments)
+        img_processor: type of ImageProcessor.
+        tokenizer: to tokenize strings.
+        use_roberta_tokenizer: Whether the tokenizer is originated from RoBerta tokenizer
+            (True by default).
+        ocr_utils: a tool to preprocess ocr_infos.
+        width: default width. It can be used only when all the images are of the same shape.
+        height: default height. It can be used only when all the images are of the same shape.
+
+    In `serialize_from_tokens`, the 3 sequences (i.e., `input_ids`, `bboxes_line`, `bboxes_word`)
+        must not contain special tokens like [CLS] or [SEP].
+    The boxes in `bboxes_line` and `bboxes_word` can be presented by either 2 points or 4 points.
+    The value in boxes should keep original.
+    Here is an example of the 3 arguments:
+        ```
+        input_ids ->
+        [[6, 2391, 6, 31833, 6, 10132, 6, 2283, 6, 17730, 6, 2698, 152]]
+        bboxes_line ->
+        [[[230, 1, 353, 38], [230, 1, 353, 38], [230, 1, 353, 38], [230, 1, 353, 38],
+            [230, 1, 353, 38], [230, 1, 353, 38], [230, 1, 353, 38], [230, 1, 353, 38],
+            [257, 155, 338, 191], [257, 155, 338, 191], [257, 155, 338, 191], [257, 155, 338, 191],
+            [257, 155, 338, 191]]]
+        bboxes_word ->
+        [[[231, 2, 267, 2, 267, 38, 231, 38], [231, 2, 267, 2, 267, 38, 231, 38],
+            [264, 7, 298, 7, 298, 36, 264, 36], [264, 7, 298, 7, 298, 36, 264, 36],
+            [293, 3, 329, 3, 329, 41, 293, 41], [293, 3, 329, 3, 329, 41, 293, 41],
+            [330, 4, 354, 4, 354, 39, 330, 39], [330, 4, 354, 4, 354, 39, 330, 39],
+            [258, 156, 289, 156, 289, 193, 258, 193], [258, 156, 289, 156, 289, 193, 258, 193],
+            [288, 158, 321, 158, 321, 192, 288, 192], [288, 158, 321, 158, 321, 192, 288, 192],
+            [321, 156, 336, 156, 336, 190, 321, 190]]]
+        ```
+
+    """
+
+    def __init__(self,
+                 max_seq_length,
+                 max_block_num,
+                 img_processor: ImageProcessor,
+                 tokenizer=None,
+                 use_roberta_tokenizer: bool = True,
+                 ocr_utils: OCRUtils = None,
+                 width=768,
+                 height=768,
+                 **kwargs):
+        self.img_processor = img_processor
+        self.tokenizer = tokenizer
+        self.kwargs = kwargs
+
+        self.serializer = TextLayoutSerializer(
+            max_seq_length,
+            max_block_num,
+            tokenizer,
+            width,
+            height,
+            use_roberta_tokenizer=use_roberta_tokenizer,
+            ocr_utils=ocr_utils)
+
+    def __call__(
+        self,
+        images: Union[list, np.ndarray, PIL.Image.Image, str],
+        ocr_infos: List[List] = None,
+        token_seqs: dict = None,
+        sizes_raw: list = None,
+    ):
+        img_data = self.img_processor(images)
+        images = img_data['images']
+        ocr_infos = img_data['ocr_infos'] if ocr_infos is None else ocr_infos
+        sizes_raw = img_data['sizes_raw'] if sizes_raw is None else sizes_raw
+        if token_seqs is None:
+            token_seqs = self.serializer(ocr_infos, sizes_raw=sizes_raw)
+        else:
+            token_seqs = self.serializer(
+                None, sizes_raw=sizes_raw, **token_seqs)
+        assert token_seqs is not None, 'token_seqs must not be NoneType!'
+        batch = {}
+        batch['image'] = images
+        for k, v in token_seqs.items():
+            batch[k] = token_seqs[k]
+        return batch
+
+    def serialize_from_tokens(self,
+                              images,
+                              input_ids,
+                              bboxes_line,
+                              bboxes_word,
+                              sizes_raw=None):
+        half_batch = {}
+        half_batch['input_ids'] = input_ids
+        half_batch['bboxes_line'] = bboxes_line
+        half_batch['bboxes_word'] = bboxes_word
+        return self(images, None, half_batch, sizes_raw)

modelscope/models/multi_modal/vldoc/tokenization.py

@@ -0,0 +1,89 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import os
+
+from transformers import XLMRobertaTokenizer
+
+SPIECE_UNDERLINE = '▁'
+
+
+class VLDocXLMTokenizer(XLMRobertaTokenizer):
+    """
+    Adapted from [`RobertaTokenizer`] and [`XLNetTokenizer`]. Based on
+    [SentencePiece](https://github.com/google/sentencepiece).
+
+    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+    this superclass for more information regarding those methods.
+
+    Args:
+        vocab_file (`str`):
+            Path to the vocabulary file.
+        bos_token (`str`, *optional*, defaults to `"<s>"`):
+            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.
+
+            <Tip>
+
+            When building a sequence using special tokens, this is not the token that is used for the beginning of
+            sequence. The token used is the `cls_token`.
+
+            </Tip>
+
+        eos_token (`str`, *optional*, defaults to `"</s>"`):
+            The end of sequence token.
+
+            <Tip>
+
+            When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+            The token used is the `sep_token`.
+
+            </Tip>
+
+        sep_token (`str`, *optional*, defaults to `"</s>"`):
+            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
+            sequence classification or for a text and a question for question answering. It is also used as the last
+            token of a sequence built with special tokens.
+        cls_token (`str`, *optional*, defaults to `"<s>"`):
+            The classifier token which is used when doing sequence classification (classification of the whole sequence
+            instead of per-token classification). It is the first token of the sequence when built with special tokens.
+        unk_token (`str`, *optional*, defaults to `"<unk>"`):
+            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
+            token instead.
+        pad_token (`str`, *optional*, defaults to `"<pad>"`):
+            The token used for padding, for example when batching sequences of different lengths.
+        mask_token (`str`, *optional*, defaults to `"<mask>"`):
+            The token used for masking values. This is the token used when training this model with masked language
+            modeling. This is the token which the model will try to predict.
+        cls_token_box (`List[int]`, *optional*, defaults to `[0, 0, 0, 0]`):
+            The bounding box to use for the special [CLS] token.
+        sep_token_box (`List[int]`, *optional*, defaults to `[1000, 1000, 1000, 1000]`):
+            The bounding box to use for the special [SEP] token.
+        pad_token_box (`List[int]`, *optional*, defaults to `[0, 0, 0, 0]`):
+            The bounding box to use for the special [PAD] token.
+        pad_token_label (`int`, *optional*, defaults to -100):
+            The label to use for padding tokens. Defaults to -100, which is the `ignore_index` of PyTorch's
+            CrossEntropyLoss.
+        only_label_first_subword (`bool`, *optional*, defaults to `True`):
+            Whether or not to only label the first subword, in case word labels are provided.
+        additional_special_tokens (`List[str]`, *optional*, defaults to `["<s>NOTUSED", "</s>NOTUSED"]`):
+            Additional special tokens used by the tokenizer.
+        sp_model_kwargs (`dict`, *optional*):
+            Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+            SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+            to set:
+
+            - `enable_sampling`: Enable subword regularization.
+            - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
+
+              - `nbest_size = {0,1}`: No sampling is performed.
+              - `nbest_size > 1`: samples from the nbest_size results.
+              - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
+                using forward-filtering-and-backward-sampling algorithm.
+
+            - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
+              BPE-dropout.
+
+    Attributes:
+        sp_model (`SentencePieceProcessor`):
+            The *SentencePiece* processor that is used for every conversion (string, tokens and IDs).
+    """
+    model_input_names = ['input_ids', 'attention_mask']

modelscope/models/multi_modal/vldoc/transformer_local.py

@@ -0,0 +1,204 @@
+# The implementation is borrowed and modified from the official PyTorch website and ABINet:
+# https://pytorch.org/docs/stable/_modules/torch/nn/modules/transformer.html
+# https://github.com/FangShancheng/ABINet/blob/main/modules/transformer.py
+
+import copy
+
+import torch.nn as nn
+from torch import Tensor
+from torch.nn import Dropout, LayerNorm, Linear, Module, ModuleList
+from torch.nn import functional as F
+
+
+class TransformerDecoder(Module):
+    r"""TransformerDecoder is a stack of N decoder layers
+
+    Args:
+        decoder_layer: an instance of the TransformerDecoderLayer() class (required).
+        num_layers: the number of sub-decoder-layers in the decoder (required).
+        norm: the layer normalization component (optional).
+
+    Examples::
+        >>> decoder_layer = nn.TransformerDecoderLayer(d_model=512, nhead=8)
+        >>> transformer_decoder = nn.TransformerDecoder(decoder_layer, num_layers=6)
+        >>> memory = torch.rand(10, 32, 512)
+        >>> tgt = torch.rand(20, 32, 512)
+        >>> out = transformer_decoder(tgt, memory)
+    """
+    __constants__ = ['norm']
+
+    def __init__(self, decoder_layer, num_layers, norm=None):
+        super(TransformerDecoder, self).__init__()
+        self.layers = _get_clones(decoder_layer, num_layers)
+        self.num_layers = num_layers
+        self.norm = norm
+
+    def forward(self,
+                tgt,
+                memory,
+                memory2=None,
+                tgt_mask=None,
+                memory_mask=None,
+                memory_mask2=None,
+                tgt_key_padding_mask=None,
+                memory_key_padding_mask=None,
+                memory_key_padding_mask2=None):
+        r"""Pass the inputs (and mask) through the decoder layer in turn.
+
+        Args:
+            tgt: the sequence to the decoder (required).
+            memory: the sequence from the last layer of the encoder (required).
+            tgt_mask: the mask for the tgt sequence (optional).
+            memory_mask: the mask for the memory sequence (optional).
+            tgt_key_padding_mask: the mask for the tgt keys per batch (optional).
+            memory_key_padding_mask: the mask for the memory keys per batch (optional).
+
+        Shape:
+            see the docs in Transformer class.
+        """
+        output = tgt
+
+        for mod in self.layers:
+            output = mod(
+                output,
+                memory,
+                memory2=memory2,
+                tgt_mask=tgt_mask,
+                memory_mask=memory_mask,
+                memory_mask2=memory_mask2,
+                tgt_key_padding_mask=tgt_key_padding_mask,
+                memory_key_padding_mask=memory_key_padding_mask,
+                memory_key_padding_mask2=memory_key_padding_mask2)
+
+        if self.norm is not None:
+            output = self.norm(output)
+
+        return output
+
+
+class TransformerDecoderLayer(Module):
+    r"""TransformerDecoderLayer is made up of self-attn, multi-head-attn and feedforward network.
+
+    Args:
+        d_model: the number of expected features in the input (required).
+        nhead: the number of heads in the multiheadattention models (required).
+        dim_feedforward: the dimension of the feedforward network model (default=2048).
+        dropout: the dropout value (default=0.1).
+        activation: the activation function of intermediate layer, relu or gelu (default=relu).
+    """
+
+    def __init__(self,
+                 d_model,
+                 nhead,
+                 dim_feedforward=2048,
+                 dropout=0.1,
+                 activation='relu',
+                 self_attn=True,
+                 siamese=False,
+                 debug=False):
+        super(TransformerDecoderLayer, self).__init__()
+        self.has_self_attn, self.siamese = self_attn, siamese
+        self.debug = debug
+        if self.has_self_attn:
+            self.self_attn = nn.MultiheadAttention(
+                d_model, nhead, dropout=dropout)
+            self.norm1 = LayerNorm(d_model)
+            self.dropout1 = Dropout(dropout)
+        self.multihead_attn = nn.MultiheadAttention(
+            d_model, nhead, dropout=dropout)
+        # Implementation of Feedforward model
+        self.linear1 = Linear(d_model, dim_feedforward)
+        self.dropout = Dropout(dropout)
+        self.linear2 = Linear(dim_feedforward, d_model)
+
+        self.norm2 = LayerNorm(d_model)
+        self.norm3 = LayerNorm(d_model)
+        self.dropout2 = Dropout(dropout)
+        self.dropout3 = Dropout(dropout)
+        if self.siamese:
+            self.multihead_attn2 = nn.MultiheadAttention(
+                d_model, nhead, dropout=dropout)
+
+        self.activation = _get_activation_fn(activation)
+
+    def __setstate__(self, state):
+        if 'activation' not in state:
+            state['activation'] = F.relu
+        super(TransformerDecoderLayer, self).__setstate__(state)
+
+    def forward(self,
+                tgt,
+                memory,
+                tgt_mask=None,
+                memory_mask=None,
+                tgt_key_padding_mask=None,
+                memory_key_padding_mask=None,
+                memory2=None,
+                memory_mask2=None,
+                memory_key_padding_mask2=None):
+        r"""Pass the inputs (and mask) through the decoder layer.
+
+        Args:
+            tgt: the sequence to the decoder layer (required).
+            memory: the sequence from the last layer of the encoder (required).
+            tgt_mask: the mask for the tgt sequence (optional).
+            memory_mask: the mask for the memory sequence (optional).
+            tgt_key_padding_mask: the mask for the tgt keys per batch (optional).
+            memory_key_padding_mask: the mask for the memory keys per batch (optional).
+
+        Shape:
+            see the docs in Transformer class.
+        """
+        if self.has_self_attn:
+            tgt2, attn = self.self_attn(
+                tgt,
+                tgt,
+                tgt,
+                attn_mask=tgt_mask,
+                key_padding_mask=tgt_key_padding_mask,
+            )
+            tgt = tgt + self.dropout1(tgt2)
+            tgt = self.norm1(tgt)
+            if self.debug:
+                self.attn = attn
+        tgt2, attn2 = self.multihead_attn(
+            tgt,
+            memory,
+            memory,
+            attn_mask=memory_mask,
+            key_padding_mask=memory_key_padding_mask)
+        if self.debug:
+            self.attn2 = attn2
+
+        if self.siamese:
+            tgt3, attn3 = self.multihead_attn2(
+                tgt,
+                memory2,
+                memory2,
+                attn_mask=memory_mask2,
+                key_padding_mask=memory_key_padding_mask2)
+            tgt = tgt + self.dropout2(tgt3)
+            if self.debug:
+                self.attn3 = attn3
+
+        tgt = tgt + self.dropout2(tgt2)
+        tgt = self.norm2(tgt)
+        tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt))))
+        tgt = tgt + self.dropout3(tgt2)
+        tgt = self.norm3(tgt)
+
+        return tgt
+
+
+def _get_clones(module, N):
+    return ModuleList([copy.deepcopy(module) for i in range(N)])
+
+
+def _get_activation_fn(activation):
+    if activation == 'relu':
+        return F.relu
+    elif activation == 'gelu':
+        return F.gelu
+
+    raise RuntimeError(
+        'activation should be relu/gelu, not {}'.format(activation))

modelscope/outputs/outputs.py

@@ -74,6 +74,14 @@ TASK_OUTPUTS = {
     # }
     Tasks.ocr_recognition: [OutputKeys.TEXT],
 
+    # document vl embedding for single sample
+    # {
+    #    "img_embedding": np.array with shape [M, D],
+    #    "text_embedding": np.array with shape [N, D]
+    # }
+    Tasks.document_vl_embedding:
+    [OutputKeys.IMG_EMBEDDING, OutputKeys.TEXT_EMBEDDING],
+
     # face 2d keypoint result for single sample
     #   {
     #       "keypoints": [
@@ -346,8 +354,9 @@ TASK_OUTPUTS = {
     #   "output_video": "path_to_rendered_video" , this is optional
     # and is only avaialbe when the "render" option is enabled.
     # }
-    Tasks.body_3d_keypoints:
-    [OutputKeys.KEYPOINTS, OutputKeys.TIMESTAMPS, OutputKeys.OUTPUT_VIDEO],
+    Tasks.body_3d_keypoints: [
+        OutputKeys.KEYPOINTS, OutputKeys.TIMESTAMPS, OutputKeys.OUTPUT_VIDEO
+    ],
 
     # 2D hand keypoints result for single sample
     # {

modelscope/pipelines/builder.py

@@ -91,6 +91,9 @@ DEFAULT_MODEL_FOR_PIPELINE = {
     Tasks.table_recognition:
     (Pipelines.table_recognition,
      'damo/cv_dla34_table-structure-recognition_cycle-centernet'),
+    Tasks.document_vl_embedding:
+    (Pipelines.document_vl_embedding,
+     'damo/multi-modal_convnext-roberta-base_vldoc-embedding'),
     Tasks.license_plate_detection:
     (Pipelines.license_plate_detection,
      'damo/cv_resnet18_license-plate-detection_damo'),
@@ -245,9 +248,9 @@ DEFAULT_MODEL_FOR_PIPELINE = {
     Tasks.video_object_segmentation:
     (Pipelines.video_object_segmentation,
      'damo/cv_rdevos_video-object-segmentation'),
-    Tasks.image_multi_view_depth_estimation:
-    (Pipelines.image_multi_view_depth_estimation,
-     'damo/cv_casmvs_multi-view-depth-estimation_general'),
+    Tasks.image_multi_view_depth_estimation: (
+        Pipelines.image_multi_view_depth_estimation,
+        'damo/cv_casmvs_multi-view-depth-estimation_general'),
 }
 
 

modelscope/pipelines/multi_modal/__init__.py

@@ -14,6 +14,7 @@ if TYPE_CHECKING:
         VideoMultiModalEmbeddingPipeline
     from .visual_question_answering_pipeline import VisualQuestionAnsweringPipeline
     from .asr_pipeline import AutomaticSpeechRecognitionPipeline
+    from .document_vl_embedding_pipeline import DocumentVLEmbeddingPipeline
     from .video_captioning_pipeline import VideoCaptioningPipeline
     from .video_question_answering_pipeline import VideoQuestionAnsweringPipeline
 else:
@@ -30,6 +31,7 @@ else:
         'generative_multi_modal_embedding_pipeline':
         ['GEMMMultiModalEmbeddingPipeline'],
         'asr_pipeline': ['AutomaticSpeechRecognitionPipeline'],
+        'document_vl_embedding_pipeline': ['DocumentVLEmbeddingPipeline'],
         'video_captioning_pipeline': ['VideoCaptioningPipeline'],
         'video_question_answering_pipeline':
         ['VideoQuestionAnsweringPipeline']

modelscope/pipelines/multi_modal/document_vl_embedding_pipeline.py

@@ -0,0 +1,62 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import Any, Dict, Optional, Union
+
+import torch
+
+from modelscope.metainfo import Pipelines
+from modelscope.models.multi_modal.vldoc.model import VLDocForDocVLEmbedding
+from modelscope.outputs import OutputKeys
+from modelscope.pipelines.base import Input, Model, Pipeline
+from modelscope.pipelines.builder import PIPELINES
+from modelscope.preprocessors.multi_modal import (Preprocessor,
+                                                  VLDocPreprocessor)
+from modelscope.utils.constant import ModelFile, Tasks
+from modelscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@PIPELINES.register_module(
+    Tasks.document_vl_embedding, module_name=Pipelines.document_vl_embedding)
+class DocumentVLEmbeddingPipeline(Pipeline):
+
+    def __init__(self,
+                 model: Union[Model, str],
+                 preprocessor: Optional[Preprocessor] = None,
+                 **kwargs):
+        """ The pipeline for multi-modal document embedding generation.
+
+        Args:
+            model: model id on modelscope hub.
+            preprocessor: type `Preprocessor`. If None, `VLDocPreprocessor` is used.
+
+        Example:
+        ```python
+        >>> from modelscope.models import Model
+        >>> from modelscope.pipelines import pipeline
+        >>> model = Model.from_pretrained(
+            'damo/multi-modal_convnext-roberta-base_vldoc-embedding')
+        >>> doc_VL_emb_pipeline = pipeline(task='document-vl-embedding', model=model)
+        >>> inp = {
+                'images': ['data/demo.png'],
+                'ocr_info_paths': ['data/demo.json']
+            }
+        >>> result = doc_VL_emb_pipeline(inp)
+        ```
+        """
+
+        super().__init__(model=model, preprocessor=preprocessor, **kwargs)
+        self.model.eval()
+        if preprocessor is None:
+            if isinstance(self.model, VLDocForDocVLEmbedding):
+                self.preprocessor = VLDocPreprocessor(self.model.model_dir)
+            else:
+                raise NotImplementedError
+
+    def forward(self, encodings: Dict[str, Any]) -> Dict[str, Any]:
+        for k, v in encodings.items():
+            encodings[k] = encodings[k].to(self.device)
+        return self.model(**encodings)
+
+    def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        return inputs

modelscope/preprocessors/multi_modal.py

@@ -390,6 +390,78 @@ class MPlugPreprocessor(Preprocessor):
             return output
 
 
+@PREPROCESSORS.register_module(
+    Fields.multi_modal, module_name=Preprocessors.vldoc_preprocessor)
+class VLDocPreprocessor(Preprocessor):
+
+    def __init__(self,
+                 model_dir: str,
+                 mode: str = ModeKeys.INFERENCE,
+                 *args,
+                 **kwargs):
+        """Preprocess data for the model `VLDocForDocVLEmbedding`.
+
+        Args:
+            model_dir (str): model path in model hub.
+            mode (str): model mode, in ('train', 'eval', 'inference').
+        """
+        super().__init__(*args, **kwargs)
+
+        self.model_dir = model_dir
+        self.mode = mode
+
+        model_cfg_path = osp.join(model_dir, 'config.json')
+        with open(model_cfg_path, 'r', encoding='utf-8') as f:
+            model_cfg = json.load(f)
+
+        from modelscope.models.multi_modal.vldoc.tokenization import VLDocXLMTokenizer
+        tokenizer_path = osp.join(model_dir, ModelFile.TOKENIZER_FOLDER)
+        self.tokenizer = VLDocXLMTokenizer.from_pretrained(tokenizer_path)
+
+        from modelscope.models.multi_modal.vldoc.processing import Processor, ImageProcessor
+        self.img_proc = ImageProcessor(
+            do_preprocess=True,
+            do_resize=True,
+            image_size={
+                'height': model_cfg['image_size'][0],
+                'width': model_cfg['image_size'][1],
+            },
+            do_normalize=True,
+            apply_ocr=False)
+        self.proc = Processor(
+            max_seq_length=model_cfg['max_seq_length'],
+            max_block_num=model_cfg['max_block_num'],
+            img_processor=self.img_proc,
+            tokenizer=self.tokenizer,
+            width=model_cfg['image_size'][1],
+            height=model_cfg['image_size'][0],
+        )
+
+    def __call__(self, input: Dict[str, Any], *args,
+                 **kwargs) -> Dict[str, Any]:
+        """
+        Args:
+            input: {
+                'images': ['img_path1', 'img_path2', ...],
+                'ocr_info_paths': ['json_path1', 'json_path2', ...]
+            }
+        Return:
+            encodings: Dict[str, Tensor]
+        """
+
+        ocr_infos = []
+        for one_ocr_info_path in input['ocr_info_paths']:
+            with open(one_ocr_info_path, 'r') as f:
+                ocr_info = json.load(f)
+                ocr_info = ocr_info['form']
+                ocr_infos.append(ocr_info)
+
+        proc_input = {'images': input['images'], 'ocr_infos': ocr_infos}
+        encodings = self.proc(**proc_input)
+
+        return encodings
+
+
 @PREPROCESSORS.register_module(
     Fields.multi_modal, module_name=Preprocessors.hitea_tasks_preprocessor)
 class HiTeAPreprocessor(Preprocessor):

modelscope/utils/constant.py

@@ -176,6 +176,7 @@ class MultiModalTasks(object):
     visual_entailment = 'visual-entailment'
     video_multi_modal_embedding = 'video-multi-modal-embedding'
     image_text_retrieval = 'image-text-retrieval'
+    document_vl_embedding = 'document-vl-embedding'
     video_captioning = 'video-captioning'
     video_question_answering = 'video-question-answering'
 
@@ -314,6 +315,7 @@ class ModelFile(object):
     LABEL_MAPPING = 'label_mapping.json'
     TRAIN_OUTPUT_DIR = 'output'
     TS_MODEL_FILE = 'model.ts'
+    TOKENIZER_FOLDER = 'tokenizer'
 
 
 class Invoke(object):

tests/pipelines/test_document_vl_embedding.py

@@ -0,0 +1,60 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import os.path as osp
+import unittest
+
+import json
+
+from modelscope.hub.snapshot_download import snapshot_download
+from modelscope.models import Model
+from modelscope.pipelines import pipeline
+from modelscope.pipelines.base import Pipeline
+from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
+from modelscope.utils.test_utils import test_level
+
+
+class DocumentVLEmbeddingTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.model_id = 'damo/multi-modal_convnext-roberta-base_vldoc-embedding'
+        cache_path = snapshot_download(self.model_id)
+        self.test_image = osp.join(cache_path, 'data/demo.png')
+        self.test_json = osp.join(cache_path, 'data/demo.json')
+        self.task = Tasks.document_vl_embedding
+
+    def pipeline_inference(self, pipe: Pipeline):
+        inp = {'images': [self.test_image], 'ocr_info_paths': [self.test_json]}
+        result = pipe(inp)
+
+        print('Results of VLDoc: ')
+        for k, v in result.items():
+            print(f'{k}: {v.size()}')
+
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_run_with_model_name(self):
+        doc_VL_emb_pipeline = pipeline(task=self.task, model=self.model_id)
+        self.pipeline_inference(doc_VL_emb_pipeline)
+
+    @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
+    def test_run_with_model_from_modelhub(self):
+        print('test_run_with_model_from_modelhub')
+        model = Model.from_pretrained(self.model_id)
+
+        doc_VL_emb_pipeline = pipeline(task=self.task, model=model)
+        self.pipeline_inference(doc_VL_emb_pipeline)
+
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_run_modelhub_default_model(self):
+        print('test_run_modelhub_default_model')
+        # default model: VLDoc
+        vldoc_doc_VL_emb_pipeline = pipeline(self.task)
+        self.pipeline_inference(vldoc_doc_VL_emb_pipeline)
+
+    @unittest.skip('demo compatibility test is only enabled on a needed-basis')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
+
+if __name__ == '__main__':
+    unittest.main()