ok Merge branch 'master' of gitlab.alibaba-inc.com:Ali-MaaS/MaaS-lib into dev/merge_github_master_0512

modelscope/metainfo.py

@@ -116,6 +116,7 @@ class Models(object):
     bad_image_detecting = 'bad-image-detecting'
     controllable_image_generation = 'controllable-image-generation'
     longshortnet = 'longshortnet'
+    fastinst = 'fastinst'
     pedestrian_attribute_recognition = 'pedestrian-attribute-recognition'
 
     # nlp models
@@ -181,6 +182,7 @@ class Models(object):
     generic_sv = 'generic-sv'
     ecapa_tdnn_sv = 'ecapa-tdnn-sv'
     campplus_sv = 'cam++-sv'
+    scl_sd = 'scl-sd'
     rdino_tdnn_sv = 'rdino_ecapa-tdnn-sv'
     generic_lm = 'generic-lm'
 
@@ -396,7 +398,7 @@ class Pipelines(object):
     nerf_recon_acc = 'nerf-recon-acc'
     bad_image_detecting = 'bad-image-detecting'
     controllable_image_generation = 'controllable-image-generation'
-
+    fast_instance_segmentation = 'fast-instance-segmentation'
     image_quality_assessment_mos = 'image-quality-assessment-mos'
     image_quality_assessment_man = 'image-quality-assessment-man'
     image_quality_assessment_degradation = 'image-quality-assessment-degradation'
@@ -480,6 +482,7 @@ class Pipelines(object):
     vad_inference = 'vad-inference'
     speaker_verification = 'speaker-verification'
     speaker_verification_rdino = 'speaker-verification-rdino'
+    speaker_change_locating = 'speaker-change-locating'
     lm_inference = 'language-score-prediction'
     speech_timestamp_inference = 'speech-timestamp-inference'
 

modelscope/models/audio/sv/DTDNN.py

@@ -76,11 +76,13 @@ class CAMPPlus(nn.Module):
                  bn_size=4,
                  init_channels=128,
                  config_str='batchnorm-relu',
-                 memory_efficient=True):
+                 memory_efficient=True,
+                 output_level='segment'):
         super(CAMPPlus, self).__init__()
 
         self.head = FCM(feat_dim=feat_dim)
         channels = self.head.out_channels
+        self.output_level = output_level
 
         self.xvector = nn.Sequential(
             OrderedDict([
@@ -118,10 +120,14 @@ class CAMPPlus(nn.Module):
         self.xvector.add_module('out_nonlinear',
                                 get_nonlinear(config_str, channels))
 
-        self.xvector.add_module('stats', StatsPool())
-        self.xvector.add_module(
-            'dense',
-            DenseLayer(channels * 2, embedding_size, config_str='batchnorm_'))
+        if self.output_level == 'segment':
+            self.xvector.add_module('stats', StatsPool())
+            self.xvector.add_module(
+                'dense',
+                DenseLayer(
+                    channels * 2, embedding_size, config_str='batchnorm_'))
+        else:
+            assert self.output_level == 'frame', '`output_level` should be set to \'segment\' or \'frame\'. '
 
         for m in self.modules():
             if isinstance(m, (nn.Conv1d, nn.Linear)):
@@ -133,6 +139,8 @@ class CAMPPlus(nn.Module):
         x = x.permute(0, 2, 1)  # (B,T,F) => (B,F,T)
         x = self.head(x)
         x = self.xvector(x)
+        if self.output_level == 'frame':
+            x = x.transpose(1, 2)
         return x
 
 

modelscope/models/audio/sv/speaker_change_locator.py

@@ -0,0 +1,319 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import os
+from collections import OrderedDict
+from typing import Any, Dict, Union
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torchaudio.compliance.kaldi as Kaldi
+
+from modelscope.metainfo import Models
+from modelscope.models import MODELS, TorchModel
+from modelscope.models.audio.sv.DTDNN import CAMPPlus
+from modelscope.utils.constant import Tasks
+
+
+class MultiHeadSelfAttention(nn.Module):
+
+    def __init__(self, n_units, h=8, dropout=0.1):
+        super(MultiHeadSelfAttention, self).__init__()
+        self.linearQ = nn.Linear(n_units, n_units)
+        self.linearK = nn.Linear(n_units, n_units)
+        self.linearV = nn.Linear(n_units, n_units)
+        self.linearO = nn.Linear(n_units, n_units)
+        self.d_k = n_units // h
+        self.h = h
+        self.dropout = nn.Dropout(p=dropout)
+        self.att = None
+
+    def forward(self, x, batch_size):
+        # x: (BT, F)
+        q = self.linearQ(x).reshape(batch_size, -1, self.h, self.d_k)
+        k = self.linearK(x).reshape(batch_size, -1, self.h, self.d_k)
+        v = self.linearV(x).reshape(batch_size, -1, self.h, self.d_k)
+        scores = torch.matmul(q.transpose(1, 2), k.permute(
+            0, 2, 3, 1)) / np.sqrt(self.d_k)
+        # scores: (B, h, T, T)
+        self.att = F.softmax(scores, dim=3)
+        p_att = self.dropout(self.att)
+        # v : (B, T, h, d_k)
+        # p_att : (B, h, T, T)
+        x = torch.matmul(p_att, v.transpose(1, 2))
+        # x : (B, h, T, d_k)
+        x = x.transpose(1, 2).reshape(-1, self.h * self.d_k)
+        return self.linearO(x)
+
+
+class PositionwiseFeedForward(nn.Module):
+
+    def __init__(self, n_units, d_units, dropout):
+        super(PositionwiseFeedForward, self).__init__()
+        self.linear1 = nn.Linear(n_units, d_units)
+        self.linear2 = nn.Linear(d_units, n_units)
+        self.dropout = nn.Dropout(p=dropout)
+
+    def forward(self, x):
+        return self.linear2(self.dropout(F.relu(self.linear1(x))))
+
+
+class PosEncoding(nn.Module):
+
+    def __init__(self, max_seq_len, d_word_vec):
+        super(PosEncoding, self).__init__()
+        pos_enc = np.array([[
+            pos / np.power(10000, 2.0 * (j // 2) / d_word_vec)
+            for j in range(d_word_vec)
+        ] for pos in range(max_seq_len)])
+        pos_enc[:, 0::2] = np.sin(pos_enc[:, 0::2])
+        pos_enc[:, 1::2] = np.cos(pos_enc[:, 1::2])
+        pad_row = np.zeros([1, d_word_vec])
+        pos_enc = np.concatenate([pad_row, pos_enc]).astype(np.float32)
+
+        self.pos_enc = torch.nn.Embedding(max_seq_len + 1, d_word_vec)
+        self.pos_enc.weight = torch.nn.Parameter(
+            torch.from_numpy(pos_enc), requires_grad=False)
+
+    def forward(self, input_len):
+        max_len = torch.max(input_len)
+        input_pos = torch.LongTensor([
+            list(range(1, len + 1)) + [0] * (max_len - len)
+            for len in input_len
+        ])
+
+        return self.pos_enc(input_pos)
+
+
+class TransformerEncoder(nn.Module):
+
+    def __init__(self,
+                 idim,
+                 n_units=256,
+                 n_layers=2,
+                 e_units=512,
+                 h=4,
+                 dropout=0.1):
+        super(TransformerEncoder, self).__init__()
+        self.linear_in = nn.Linear(idim, n_units)
+        self.lnorm_in = nn.LayerNorm(n_units)
+
+        self.n_layers = n_layers
+        self.dropout = nn.Dropout(p=dropout)
+        for i in range(n_layers):
+            setattr(self, '{}{:d}'.format('lnorm1_', i), nn.LayerNorm(n_units))
+            setattr(self, '{}{:d}'.format('self_att_', i),
+                    MultiHeadSelfAttention(n_units, h))
+            setattr(self, '{}{:d}'.format('lnorm2_', i), nn.LayerNorm(n_units))
+            setattr(self, '{}{:d}'.format('ff_', i),
+                    PositionwiseFeedForward(n_units, e_units, dropout))
+        self.lnorm_out = nn.LayerNorm(n_units)
+
+    def forward(self, x):
+        # x: [B, num_anchors, T, n_in]
+        bs, num, tframe, dim = x.size()
+        x = x.reshape(bs * num, tframe, -1)  # [B*num_anchors, T, dim]
+        # x: (B, T, F) ... batch, time, (mel)freq
+        B_size, T_size, _ = x.shape
+        # e: (BT, F)
+        e = self.linear_in(x.reshape(B_size * T_size, -1))
+        # Encoder stack
+        for i in range(self.n_layers):
+            # layer normalization
+            e = getattr(self, '{}{:d}'.format('lnorm1_', i))(e)
+            # self-attention
+            s = getattr(self, '{}{:d}'.format('self_att_', i))(e, x.shape[0])
+            # residual
+            e = e + self.dropout(s)
+            # layer normalization
+            e = getattr(self, '{}{:d}'.format('lnorm2_', i))(e)
+            # positionwise feed-forward
+            s = getattr(self, '{}{:d}'.format('ff_', i))(e)
+            # residual
+            e = e + self.dropout(s)
+        # final layer normalization
+        # output: (BT, F)
+        # output: (B, F, T)
+        output = self.lnorm_out(e).reshape(B_size, T_size, -1)
+        output = output.reshape(bs, num, tframe,
+                                -1)  # [B, num_anchors, T, dim]
+        return output
+
+
+class TransformerEncoder_out(nn.Module):
+
+    def __init__(self,
+                 idim,
+                 n_units=256,
+                 n_layers=2,
+                 e_units=512,
+                 h=4,
+                 dropout=0.1):
+        super(TransformerEncoder_out, self).__init__()
+        self.linear_in = nn.Linear(idim, n_units)
+        self.lnorm_in = nn.LayerNorm(n_units)
+
+        self.n_layers = n_layers
+        self.dropout = nn.Dropout(p=dropout)
+        for i in range(n_layers):
+            setattr(self, '{}{:d}'.format('lnorm1_', i), nn.LayerNorm(n_units))
+            setattr(self, '{}{:d}'.format('self_att_', i),
+                    MultiHeadSelfAttention(n_units, h))
+            setattr(self, '{}{:d}'.format('lnorm2_', i), nn.LayerNorm(n_units))
+            setattr(self, '{}{:d}'.format('ff_', i),
+                    PositionwiseFeedForward(n_units, e_units, dropout))
+        self.lnorm_out = nn.LayerNorm(n_units)
+
+    def forward(self, x):
+        # x: (B, T, F)
+        B_size, T_size, _ = x.shape
+        # e: (BT, F)
+        e = self.linear_in(x.reshape(B_size * T_size, -1))
+        # Encoder stack
+        for i in range(self.n_layers):
+            # layer normalization
+            e = getattr(self, '{}{:d}'.format('lnorm1_', i))(e)
+            # self-attention
+            s = getattr(self, '{}{:d}'.format('self_att_', i))(e, x.shape[0])
+            # residual
+            e = e + self.dropout(s)
+            # layer normalization
+            e = getattr(self, '{}{:d}'.format('lnorm2_', i))(e)
+            # positionwise feed-forward
+            s = getattr(self, '{}{:d}'.format('ff_', i))(e)
+            # residual
+            e = e + self.dropout(s)
+        # final layer normalization
+        # output: (BT, F)
+        # output: (B, T, F)
+        output = self.lnorm_out(e).reshape(B_size, T_size, -1)
+        return output
+
+
+class OutLayer(nn.Module):
+
+    def __init__(self, n_units=256, num_anchors=2):
+        super(OutLayer, self).__init__()
+        self.combine = TransformerEncoder_out(num_anchors * n_units, n_units)
+        self.out_linear = nn.Linear(n_units // num_anchors, 1)
+
+    def forward(self, input):
+        # input: [B, num_anchors, T, dim]
+        bs, num, tframe, dim = input.size()
+        output = input.permute(0, 2, 1,
+                               3).reshape(bs, tframe,
+                                          -1)  # [Bs, t, num_anchors*dim]
+        output = self.combine(output)  # [Bs, t, n_units]
+        output = output.reshape(
+            bs, tframe, num, -1)  # [Bs, t, num_anchors, n_units//num_anchors]
+        output = self.out_linear(output).squeeze(-1)  # [Bs, t, num_anchors]
+
+        return output
+
+
+class TransformerDetector(nn.Module):
+
+    def __init__(self,
+                 frame_dim=512,
+                 anchor_dim=192,
+                 hidden_dim=256,
+                 max_seq_len=1000):
+        super(TransformerDetector, self).__init__()
+        self.detection = TransformerEncoder(
+            idim=frame_dim + anchor_dim, n_units=hidden_dim)
+        self.output = OutLayer(n_units=hidden_dim)
+        self.pos_enc = PosEncoding(max_seq_len, hidden_dim)
+
+    def forward(self, feats, anchors):
+        # feats: [1, t, fdim]
+        num_frames = feats.shape[1]
+        num_anchors = anchors.shape[1]
+        bs = feats.shape[0]
+        feats = feats.unsqueeze(1).repeat(
+            1, num_anchors, 1, 1)  # shape: [Bs, num_anchors, t, fdim]
+        anchors = anchors.unsqueeze(2).repeat(
+            1, 1, num_frames, 1)  # shape: [Bs, num_anchors, t, xdim]
+        sd_in = torch.cat((feats, anchors),
+                          dim=-1)  # shape: [Bs, num_anchors, t, fdim+xdim]
+        sd_out = self.detection(sd_in)  # shape: [Bs, num_anchors, t, sd_dim]
+
+        # pos
+        pos_emb = self.pos_enc(torch.tensor([num_frames] * (bs * num_anchors)))
+        pos_emb = pos_emb.reshape(bs, num_anchors, num_frames, -1)
+        sd_out += pos_emb
+
+        # output
+        output = self.output(sd_out)  # shape: [Bs, t, num_anchors]
+
+        return output
+
+
+@MODELS.register_module(Tasks.speaker_diarization, module_name=Models.scl_sd)
+class SpeakerChangeLocatorTransformer(TorchModel):
+    r"""A speaekr change locator using the transformer architecture as the backbone.
+    Args:
+        model_dir: A model dir.
+        model_config: The model config.
+    """
+
+    def __init__(self, model_dir, model_config: Dict[str, Any], *args,
+                 **kwargs):
+        super().__init__(model_dir, model_config, *args, **kwargs)
+        self.model_config = model_config
+
+        self.feature_dim = self.model_config['fbank_dim']
+        frame_size = self.model_config['frame_size']
+        anchor_size = self.model_config['anchor_size']
+
+        self.encoder = CAMPPlus(self.feature_dim, output_level='frame')
+        self.backend = TransformerDetector(
+            frame_dim=frame_size, anchor_dim=anchor_size)
+
+        pretrained_encoder = kwargs['pretrained_encoder']
+        pretrained_backend = kwargs['pretrained_backend']
+
+        self.__load_check_point(pretrained_encoder, pretrained_backend)
+
+        self.encoder.eval()
+        self.backend.eval()
+
+    def forward(self, audio, anchors):
+        assert len(audio.shape) == 2 and audio.shape[
+            0] == 1, 'modelscope error: the shape of input audio to model needs to be [1, T]'
+        assert len(
+            anchors.shape
+        ) == 3 and anchors.shape[0] == 1 and anchors.shape[
+            1] == 2, 'modelscope error: the shape of input anchors to model needs to be [1, 2, D]'
+        # audio shape: [1, T]
+        feature = self.__extract_feature(audio)
+        frame_state = self.encoder(feature)
+        output = self.backend(frame_state, anchors)
+        output = output.squeeze(0).detach().cpu().sigmoid()
+
+        time_scale_factor = int(np.ceil(feature.shape[1] / output.shape[0]))
+        output = output.unsqueeze(1).expand(-1, time_scale_factor,
+                                            -1).reshape(-1, output.shape[-1])
+        return output
+
+    def __extract_feature(self, audio):
+        feature = Kaldi.fbank(audio, num_mel_bins=self.feature_dim)
+        feature = feature - feature.mean(dim=0, keepdim=True)
+        feature = feature.unsqueeze(0)
+        return feature
+
+    def __load_check_point(self,
+                           pretrained_encoder,
+                           pretrained_backend,
+                           device=None):
+        if not device:
+            device = torch.device('cpu')
+        self.encoder.load_state_dict(
+            torch.load(
+                os.path.join(self.model_dir, pretrained_encoder),
+                map_location=device))
+
+        self.backend.load_state_dict(
+            torch.load(
+                os.path.join(self.model_dir, pretrained_backend),
+                map_location=device))

modelscope/models/cv/image_instance_segmentation/__init__.py

@@ -8,10 +8,12 @@ if TYPE_CHECKING:
     from .maskdino_swin import MaskDINOSwin
     from .model import CascadeMaskRCNNSwinModel
     from .maskdino_model import MaskDINOSwinModel
+    from .fastinst_model import FastInst
     from .postprocess_utils import get_img_ins_seg_result, get_maskdino_ins_seg_result
 else:
     _import_structure = {
         'cascade_mask_rcnn_swin': ['CascadeMaskRCNNSwin'],
+        'fastinst_model': ['FastInst'],
         'maskdino_swin': ['MaskDINOSwin'],
         'model': ['CascadeMaskRCNNSwinModel'],
         'maskdino_model': ['MaskDINOSwinModel'],

modelscope/models/cv/image_instance_segmentation/backbones/__init__.py

@@ -6,10 +6,12 @@ from modelscope.utils.import_utils import LazyImportModule
 if TYPE_CHECKING:
     from .swin_transformer import SwinTransformer
     from .swin_transformer import D2SwinTransformer
+    from .resnet import build_resnet_backbone
 
 else:
     _import_structure = {
         'swin_transformer': ['SwinTransformer', 'D2SwinTransformer'],
+        'resnet': ['build_resnet_backbone']
     }
 
     import sys

modelscope/models/cv/image_instance_segmentation/backbones/resnet.py

@@ -0,0 +1,114 @@
+# Part of the implementation is borrowed and modified from Detectron2, publicly available at
+# https://github.com/facebookresearch/detectron2/blob/main/projects/DeepLab/deeplab/resnet.py
+
+import torch.nn.functional as F
+from torch import nn
+
+from modelscope.models.cv.image_human_parsing.backbone.deeplab_resnet import (
+    BottleneckBlock, DeeplabResNet, get_norm)
+from modelscope.models.cv.image_instance_segmentation.maskdino.utils import \
+    Conv2d
+
+
+class BasicStem(nn.Module):
+    """
+    The standard ResNet stem (layers before the first residual block),
+    with a conv, relu and max_pool.
+    """
+
+    def __init__(self, in_channels=3, out_channels=64, norm='BN'):
+        """
+        Args:
+            norm (str or callable): norm after the first conv layer.
+                See :func:`layers.get_norm` for supported format.
+        """
+        super().__init__()
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+        self.stride = 4
+        self.conv1 = Conv2d(
+            in_channels,
+            out_channels,
+            kernel_size=7,
+            stride=2,
+            padding=3,
+            bias=False,
+            norm=get_norm(norm, out_channels),
+        )
+
+    def forward(self, x):
+        x = self.conv1(x)
+        x = F.relu_(x)
+        x = F.max_pool2d(x, kernel_size=3, stride=2, padding=1)
+        return x
+
+
+def build_resnet_backbone(out_features, depth, num_groups, width_per_group,
+                          norm, stem_out_channels, res2_out_channels,
+                          stride_in_1x1, res4_dilation, res5_dilation,
+                          res5_multi_grid, input_shape):
+    stem = BasicStem(
+        in_channels=input_shape['channels'],
+        out_channels=stem_out_channels,
+        norm=norm)
+    bottleneck_channels = num_groups * width_per_group
+    in_channels = stem_out_channels
+    out_channels = res2_out_channels
+
+    assert res4_dilation in {
+        1, 2
+    }, 'res4_dilation cannot be {}.'.format(res4_dilation)
+    assert res5_dilation in {
+        1, 2, 4
+    }, 'res5_dilation cannot be {}.'.format(res5_dilation)
+    if res4_dilation == 2:
+        # Always dilate res5 if res4 is dilated.
+        assert res5_dilation == 4
+
+    num_blocks_per_stage = {
+        50: [3, 4, 6, 3],
+        101: [3, 4, 23, 3],
+        152: [3, 8, 36, 3]
+    }[depth]
+
+    stages = []
+    out_stage_idx = [{
+        'res2': 2,
+        'res3': 3,
+        'res4': 4,
+        'res5': 5
+    }[f] for f in out_features]
+    max_stage_idx = max(out_stage_idx)
+    for idx, stage_idx in enumerate(range(2, max_stage_idx + 1)):
+        if stage_idx == 4:
+            dilation = res4_dilation
+        elif stage_idx == 5:
+            dilation = res5_dilation
+        else:
+            dilation = 1
+        first_stride = 1 if idx == 0 or dilation > 1 else 2
+        stride_per_block = [first_stride]
+        stride_per_block += [1] * (num_blocks_per_stage[idx] - 1)
+        stage_kargs = {
+            'num_blocks': num_blocks_per_stage[idx],
+            'stride_per_block': stride_per_block,
+            'in_channels': in_channels,
+            'out_channels': out_channels,
+            'norm': norm,
+            'bottleneck_channels': bottleneck_channels,
+            'stride_in_1x1': stride_in_1x1,
+            'dilation': dilation,
+            'num_groups': num_groups,
+            'block_class': BottleneckBlock
+        }
+        if stage_idx == 5:
+            stage_kargs.pop('dilation')
+            stage_kargs['dilation_per_block'] = [
+                dilation * mg for mg in res5_multi_grid
+            ]
+        blocks = DeeplabResNet.make_stage(**stage_kargs)
+        in_channels = out_channels
+        out_channels *= 2
+        bottleneck_channels *= 2
+        stages.append(blocks)
+    return DeeplabResNet(stem, stages, out_features=out_features)

modelscope/models/cv/image_instance_segmentation/fastinst/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.

modelscope/models/cv/image_instance_segmentation/fastinst/fastinst_decoder.py

@@ -0,0 +1,351 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import math
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+from modelscope.models.cv.image_colorization.ddcolor.utils.transformer_utils import (
+    MLP, CrossAttentionLayer, FFNLayer, SelfAttentionLayer)
+
+
+class QueryProposal(nn.Module):
+
+    def __init__(self, num_features, num_queries, num_classes):
+        super().__init__()
+        self.topk = num_queries
+        self.num_classes = num_classes
+
+        self.conv_proposal_cls_logits = nn.Sequential(
+            nn.Conv2d(
+                num_features, num_features, kernel_size=3, stride=1,
+                padding=1),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(
+                num_features,
+                num_classes + 1,
+                kernel_size=1,
+                stride=1,
+                padding=0),
+        )
+
+    @torch.no_grad()
+    def compute_coordinates(self, x):
+        h, w = x.size(2), x.size(3)
+        y_loc = torch.linspace(0, 1, h, device=x.device)
+        x_loc = torch.linspace(0, 1, w, device=x.device)
+        y_loc, x_loc = torch.meshgrid(y_loc, x_loc)
+        locations = torch.stack([x_loc, y_loc], 0).unsqueeze(0)
+        return locations
+
+    def seek_local_maximum(self, x, epsilon=1e-6):
+        """
+        inputs:
+            x: torch.tensor, shape [b, c, h, w]
+        return:
+            torch.tensor, shape [b, c, h, w]
+        """
+        x_pad = F.pad(x, (1, 1, 1, 1), 'constant', 0)
+        # top, bottom, left, right, top-left, top-right, bottom-left, bottom-right
+        maximum = (x >= x_pad[:, :, :-2, 1:-1]) & \
+                  (x >= x_pad[:, :, 2:, 1:-1]) & \
+                  (x >= x_pad[:, :, 1:-1, :-2]) & \
+                  (x >= x_pad[:, :, 1:-1, 2:]) & \
+                  (x >= x_pad[:, :, :-2, :-2]) & \
+                  (x >= x_pad[:, :, :-2, 2:]) & \
+                  (x >= x_pad[:, :, 2:, :-2]) & \
+                  (x >= x_pad[:, :, 2:, 2:]) & \
+                  (x >= epsilon)
+        return maximum.to(x)
+
+    def forward(self, x, pos_embeddings):
+
+        proposal_cls_logits = self.conv_proposal_cls_logits(x)  # b, c, h, w
+        proposal_cls_probs = proposal_cls_logits.softmax(dim=1)  # b, c, h, w
+        proposal_cls_one_hot = F.one_hot(
+            proposal_cls_probs[:, :-1, :, :].max(1)[1],
+            num_classes=self.num_classes + 1).permute(0, 3, 1, 2)  # b, c, h, w
+        proposal_cls_probs = proposal_cls_probs.mul(proposal_cls_one_hot)
+        proposal_local_maximum_map = self.seek_local_maximum(
+            proposal_cls_probs)  # b, c, h, w
+        proposal_cls_probs = proposal_cls_probs + proposal_local_maximum_map  # b, c, h, w
+
+        # top-k indices
+        topk_indices = torch.topk(
+            proposal_cls_probs[:, :-1, :, :].flatten(2).max(1)[0],
+            self.topk,
+            dim=1)[1]  # b, q
+        topk_indices = topk_indices.unsqueeze(1)  # b, 1, q
+
+        # topk queries
+        topk_proposals = torch.gather(
+            x.flatten(2), dim=2, index=topk_indices.repeat(1, x.shape[1],
+                                                           1))  # b, c, q
+        pos_embeddings = pos_embeddings.repeat(x.shape[0], 1, 1, 1).flatten(2)
+        topk_pos_embeddings = torch.gather(
+            pos_embeddings,
+            dim=2,
+            index=topk_indices.repeat(1, pos_embeddings.shape[1],
+                                      1))  # b, c, q
+        if self.training:
+            locations = self.compute_coordinates(x).repeat(x.shape[0], 1, 1, 1)
+            topk_locations = torch.gather(
+                locations.flatten(2),
+                dim=2,
+                index=topk_indices.repeat(1, locations.shape[1], 1))
+            topk_locations = topk_locations.transpose(-1, -2)  # b, q, 2
+        else:
+            topk_locations = None
+        return topk_proposals, topk_pos_embeddings, topk_locations, proposal_cls_logits
+
+
+class FastInstDecoder(nn.Module):
+
+    def __init__(self, in_channels, *, num_classes: int, hidden_dim: int,
+                 num_queries: int, num_aux_queries: int, nheads: int,
+                 dim_feedforward: int, dec_layers: int, pre_norm: bool,
+                 mask_dim: int):
+        """
+        Args:
+            in_channels: channels of the input features
+            num_classes: number of classes
+            hidden_dim: Transformer feature dimension
+            num_queries: number of queries
+            num_aux_queries: number of auxiliary queries
+            nheads: number of heads
+            dim_feedforward: feature dimension in feedforward network
+            dec_layers: number of Transformer decoder layers
+            pre_norm: whether to use pre-LayerNorm or not
+            mask_dim: mask feature dimension
+        """
+        super().__init__()
+        self.num_heads = nheads
+        self.num_layers = dec_layers
+        self.num_queries = num_queries
+        self.num_aux_queries = num_aux_queries
+        self.num_classes = num_classes
+
+        meta_pos_size = int(round(math.sqrt(self.num_queries)))
+        self.meta_pos_embed = nn.Parameter(
+            torch.empty(1, hidden_dim, meta_pos_size, meta_pos_size))
+        if num_aux_queries > 0:
+            self.empty_query_features = nn.Embedding(num_aux_queries,
+                                                     hidden_dim)
+            self.empty_query_pos_embed = nn.Embedding(num_aux_queries,
+                                                      hidden_dim)
+
+        self.query_proposal = QueryProposal(hidden_dim, num_queries,
+                                            num_classes)
+
+        self.transformer_query_cross_attention_layers = nn.ModuleList()
+        self.transformer_query_self_attention_layers = nn.ModuleList()
+        self.transformer_query_ffn_layers = nn.ModuleList()
+        self.transformer_mask_cross_attention_layers = nn.ModuleList()
+        self.transformer_mask_ffn_layers = nn.ModuleList()
+        for idx in range(self.num_layers):
+            self.transformer_query_cross_attention_layers.append(
+                CrossAttentionLayer(
+                    d_model=hidden_dim,
+                    nhead=nheads,
+                    dropout=0.0,
+                    normalize_before=pre_norm))
+            self.transformer_query_self_attention_layers.append(
+                SelfAttentionLayer(
+                    d_model=hidden_dim,
+                    nhead=nheads,
+                    dropout=0.0,
+                    normalize_before=pre_norm))
+            self.transformer_query_ffn_layers.append(
+                FFNLayer(
+                    d_model=hidden_dim,
+                    dim_feedforward=dim_feedforward,
+                    dropout=0.0,
+                    normalize_before=pre_norm))
+            self.transformer_mask_cross_attention_layers.append(
+                CrossAttentionLayer(
+                    d_model=hidden_dim,
+                    nhead=nheads,
+                    dropout=0.0,
+                    normalize_before=pre_norm))
+            self.transformer_mask_ffn_layers.append(
+                FFNLayer(
+                    d_model=hidden_dim,
+                    dim_feedforward=dim_feedforward,
+                    dropout=0.0,
+                    normalize_before=pre_norm))
+
+        self.decoder_query_norm_layers = nn.ModuleList()
+        self.class_embed_layers = nn.ModuleList()
+        self.mask_embed_layers = nn.ModuleList()
+        self.mask_features_layers = nn.ModuleList()
+        for idx in range(self.num_layers + 1):
+            self.decoder_query_norm_layers.append(nn.LayerNorm(hidden_dim))
+            self.class_embed_layers.append(
+                MLP(hidden_dim, hidden_dim, num_classes + 1, 3))
+            self.mask_embed_layers.append(
+                MLP(hidden_dim, hidden_dim, mask_dim, 3))
+            self.mask_features_layers.append(nn.Linear(hidden_dim, mask_dim))
+
+    def forward(self, x, mask_features, targets=None):
+        bs = x[0].shape[0]
+        proposal_size = x[1].shape[-2:]
+        pixel_feature_size = x[2].shape[-2:]
+
+        pixel_pos_embeds = F.interpolate(
+            self.meta_pos_embed,
+            size=pixel_feature_size,
+            mode='bilinear',
+            align_corners=False)
+        proposal_pos_embeds = F.interpolate(
+            self.meta_pos_embed,
+            size=proposal_size,
+            mode='bilinear',
+            align_corners=False)
+
+        pixel_features = x[2].flatten(2).permute(2, 0, 1)
+        pixel_pos_embeds = pixel_pos_embeds.flatten(2).permute(2, 0, 1)
+
+        query_features, query_pos_embeds, query_locations, proposal_cls_logits = self.query_proposal(
+            x[1], proposal_pos_embeds)
+        query_features = query_features.permute(2, 0, 1)
+        query_pos_embeds = query_pos_embeds.permute(2, 0, 1)
+        if self.num_aux_queries > 0:
+            aux_query_features = self.empty_query_features.weight.unsqueeze(
+                1).repeat(1, bs, 1)
+            aux_query_pos_embed = self.empty_query_pos_embed.weight.unsqueeze(
+                1).repeat(1, bs, 1)
+            query_features = torch.cat([query_features, aux_query_features],
+                                       dim=0)
+            query_pos_embeds = torch.cat(
+                [query_pos_embeds, aux_query_pos_embed], dim=0)
+
+        outputs_class, outputs_mask, attn_mask, _, _ = self.forward_prediction_heads(
+            query_features,
+            pixel_features,
+            pixel_feature_size,
+            -1,
+            return_attn_mask=True)
+        predictions_class = [outputs_class]
+        predictions_mask = [outputs_mask]
+        predictions_matching_index = [None]
+        query_feature_memory = [query_features]
+        pixel_feature_memory = [pixel_features]
+
+        for i in range(self.num_layers):
+            query_features, pixel_features = self.forward_one_layer(
+                query_features, pixel_features, query_pos_embeds,
+                pixel_pos_embeds, attn_mask, i)
+            if i < self.num_layers - 1:
+                outputs_class, outputs_mask, attn_mask, _, _ = self.forward_prediction_heads(
+                    query_features,
+                    pixel_features,
+                    pixel_feature_size,
+                    i,
+                    return_attn_mask=True,
+                )
+            else:
+                outputs_class, outputs_mask, _, matching_indices, gt_attn_mask = self.forward_prediction_heads(
+                    query_features,
+                    pixel_features,
+                    pixel_feature_size,
+                    i,
+                )
+            predictions_class.append(outputs_class)
+            predictions_mask.append(outputs_mask)
+            predictions_matching_index.append(None)
+            query_feature_memory.append(query_features)
+            pixel_feature_memory.append(pixel_features)
+
+        out = {
+            'proposal_cls_logits':
+            proposal_cls_logits,
+            'query_locations':
+            query_locations,
+            'pred_logits':
+            predictions_class[-1],
+            'pred_masks':
+            predictions_mask[-1],
+            'pred_indices':
+            predictions_matching_index[-1],
+            'aux_outputs':
+            self._set_aux_loss(predictions_class, predictions_mask,
+                               predictions_matching_index, query_locations)
+        }
+        return out
+
+    def forward_one_layer(self, query_features, pixel_features,
+                          query_pos_embeds, pixel_pos_embeds, attn_mask, i):
+        pixel_features = self.transformer_mask_cross_attention_layers[i](
+            pixel_features,
+            query_features,
+            query_pos=pixel_pos_embeds,
+            pos=query_pos_embeds)
+        pixel_features = self.transformer_mask_ffn_layers[i](pixel_features)
+
+        query_features = self.transformer_query_cross_attention_layers[i](
+            query_features,
+            pixel_features,
+            memory_mask=attn_mask,
+            query_pos=query_pos_embeds,
+            pos=pixel_pos_embeds)
+        query_features = self.transformer_query_self_attention_layers[i](
+            query_features, query_pos=query_pos_embeds)
+        query_features = self.transformer_query_ffn_layers[i](query_features)
+        return query_features, pixel_features
+
+    def forward_prediction_heads(self,
+                                 query_features,
+                                 pixel_features,
+                                 pixel_feature_size,
+                                 idx_layer,
+                                 return_attn_mask=False,
+                                 return_gt_attn_mask=False,
+                                 targets=None,
+                                 query_locations=None):
+        decoder_query_features = self.decoder_query_norm_layers[idx_layer + 1](
+            query_features[:self.num_queries])
+        decoder_query_features = decoder_query_features.transpose(0, 1)
+        if idx_layer + 1 == self.num_layers:
+            outputs_class = self.class_embed_layers[idx_layer + 1](
+                decoder_query_features)
+        else:
+            outputs_class = None
+        outputs_mask_embed = self.mask_embed_layers[idx_layer + 1](
+            decoder_query_features)
+        outputs_mask_features = self.mask_features_layers[idx_layer + 1](
+            pixel_features.transpose(0, 1))
+
+        outputs_mask = torch.einsum('bqc,blc->bql', outputs_mask_embed,
+                                    outputs_mask_features)
+        outputs_mask = outputs_mask.reshape(-1, self.num_queries,
+                                            *pixel_feature_size)
+
+        if return_attn_mask:
+            # outputs_mask.shape: b, q, h, w
+            attn_mask = F.pad(outputs_mask,
+                              (0, 0, 0, 0, 0, self.num_aux_queries),
+                              'constant', 1)
+            attn_mask = (attn_mask < 0.).flatten(2)  # b, q, hw
+            invalid_query = attn_mask.all(-1, keepdim=True)  # b, q, 1
+            attn_mask = (~invalid_query) & attn_mask  # b, q, hw
+            attn_mask = attn_mask.unsqueeze(1).repeat(1, self.num_heads, 1,
+                                                      1).flatten(0, 1)
+            attn_mask = attn_mask.detach()
+        else:
+            attn_mask = None
+
+        matching_indices = None
+        gt_attn_mask = None
+
+        return outputs_class, outputs_mask, attn_mask, matching_indices, gt_attn_mask
+
+    @torch.jit.unused
+    def _set_aux_loss(self, outputs_class, outputs_seg_masks, output_indices,
+                      output_query_locations):
+        return [{
+            'query_locations': output_query_locations,
+            'pred_logits': a,
+            'pred_masks': b,
+            'pred_matching_indices': c
+        } for a, b, c in zip(outputs_class[:-1], outputs_seg_masks[:-1],
+                             output_indices[:-1])]

modelscope/models/cv/image_instance_segmentation/fastinst/fastinst_encoder.py

@@ -0,0 +1,180 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import logging
+from typing import Callable, Optional, Union
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+from modelscope.models.cv.image_instance_segmentation.maskdino.utils import \
+    Conv2d
+
+
+# This is a modified FPN decoder.
+class BaseFPN(nn.Module):
+
+    def __init__(
+        self,
+        input_shape,
+        *,
+        convs_dim: int,
+        mask_dim: int,
+        norm: Optional[Union[str, Callable]] = None,
+    ):
+        """
+        Args:
+            input_shape: shapes (channels and stride) of the input features
+            convs_dim: number of output channels for the intermediate conv layers.
+            mask_dim: number of output channels for the final conv layer.
+            norm (str or callable): normalization for all conv layers
+        """
+        super().__init__()
+
+        input_shape = sorted(input_shape.items(), key=lambda x: x[1]['stride'])
+        self.in_features = [k for k, v in input_shape
+                            ]  # starting from "res3" to "res5"
+        feature_channels = [v['channels'] for k, v in input_shape]
+
+        lateral_convs = []
+        output_convs = []
+
+        use_bias = norm == ''
+        for idx, in_channels in enumerate(feature_channels):
+            lateral_norm = nn.GroupNorm(32, convs_dim)
+            output_norm = nn.GroupNorm(32, convs_dim)
+
+            lateral_conv = Conv2d(
+                in_channels,
+                convs_dim,
+                kernel_size=1,
+                bias=use_bias,
+                norm=lateral_norm)
+            output_conv = Conv2d(
+                convs_dim,
+                convs_dim,
+                kernel_size=3,
+                stride=1,
+                padding=1,
+                bias=use_bias,
+                norm=output_norm,
+                activation=F.relu,
+            )
+            self.add_module('adapter_{}'.format(idx + 1), lateral_conv)
+            self.add_module('layer_{}'.format(idx + 1), output_conv)
+
+            lateral_convs.append(lateral_conv)
+            output_convs.append(output_conv)
+        # Place convs into top-down order (from low to high resolution)
+        # to make the top-down computation in forward clearer.
+        self.lateral_convs = lateral_convs[::-1]
+        self.output_convs = output_convs[::-1]
+
+        self.convs_dim = convs_dim
+        self.num_feature_levels = 3  # always use 3 scales
+
+    def forward_features(self, features):
+        multi_scale_features = []
+        num_cur_levels = 0
+        # Reverse feature maps into top-down order (from low to high resolution)
+        for idx, f in enumerate(self.in_features[::-1]):
+            x = features[f]
+            lateral_conv = self.lateral_convs[idx]
+            output_conv = self.output_convs[idx]
+            if idx == 0:
+                y = lateral_conv(x)
+            else:
+                cur_fpn = lateral_conv(x)
+                y = cur_fpn + F.interpolate(
+                    y,
+                    size=cur_fpn.shape[-2:],
+                    mode='bilinear',
+                    align_corners=False)
+            y = output_conv(y)
+
+            if num_cur_levels < self.num_feature_levels:
+                multi_scale_features.append(y)
+                num_cur_levels += 1
+        return None, multi_scale_features
+
+    def forward(self, features, targets=None):
+        logger = logging.getLogger(__name__)
+        logger.warning(
+            'Calling forward() may cause unpredicted behavior of PixelDecoder module.'
+        )
+        return self.forward_features(features)
+
+
+class PyramidPoolingModule(nn.Module):
+
+    def __init__(self, in_channels, channels=512, sizes=(1, 2, 3, 6)):
+        super().__init__()
+        self.stages = []
+        self.stages = nn.ModuleList(
+            [self._make_stage(in_channels, channels, size) for size in sizes])
+        self.bottleneck = Conv2d(in_channels + len(sizes) * channels,
+                                 in_channels, 1)
+
+    def _make_stage(self, features, out_features, size):
+        prior = nn.AdaptiveAvgPool2d(output_size=(size, size))
+        conv = Conv2d(features, out_features, 1)
+        return nn.Sequential(prior, conv)
+
+    def forward(self, feats):
+        h, w = feats.size(2), feats.size(3)
+        priors = [
+            F.interpolate(
+                input=F.relu_(stage(feats)),
+                size=(h, w),
+                mode='bilinear',
+                align_corners=False) for stage in self.stages
+        ] + [feats]
+        out = F.relu_(self.bottleneck(torch.cat(priors, 1)))
+        return out
+
+
+class PyramidPoolingModuleFPN(BaseFPN):
+
+    def __init__(
+        self,
+        input_shape,
+        *,
+        convs_dim: int,
+        mask_dim: int,
+        norm: Optional[Union[str, Callable]] = None,
+    ):
+        """
+        NOTE: this interface is experimental.
+        Args:
+            input_shape: shapes (channels and stride) of the input features
+            convs_dim: number of output channels for the intermediate conv layers.
+            mask_dim: number of output channels for the final conv layer.
+            norm (str or callable): normalization for all conv layers
+        """
+        super().__init__(
+            input_shape, convs_dim=convs_dim, mask_dim=mask_dim, norm=norm)
+        self.ppm = PyramidPoolingModule(convs_dim, convs_dim // 4)
+
+    def forward_features(self, features):
+        multi_scale_features = []
+        num_cur_levels = 0
+        # Reverse feature maps into top-down order (from low to high resolution)
+        for idx, f in enumerate(self.in_features[::-1]):
+            x = features[f]
+            lateral_conv = self.lateral_convs[idx]
+            output_conv = self.output_convs[idx]
+            if idx == 0:
+                y = self.ppm(lateral_conv(x))
+            else:
+                cur_fpn = lateral_conv(x)
+                y = cur_fpn + F.interpolate(
+                    y,
+                    size=cur_fpn.shape[-2:],
+                    mode='bilinear',
+                    align_corners=False)
+            y = output_conv(y)
+
+            if num_cur_levels < self.num_feature_levels:
+                multi_scale_features.append(y)
+                num_cur_levels += 1
+
+        return None, multi_scale_features

modelscope/models/cv/image_instance_segmentation/fastinst_model.py

@@ -0,0 +1,221 @@
+# Part of implementation is borrowed and modified from Mask2Former, publicly available at
+# https://github.com/facebookresearch/Mask2Former.
+import os
+from typing import Any, Dict, List
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from modelscope.metainfo import Models
+from modelscope.models.base import TorchModel
+from modelscope.models.builder import MODELS
+from modelscope.models.cv.image_instance_segmentation.maskdino_swin import \
+    ImageList
+from modelscope.utils.constant import ModelFile, Tasks
+from modelscope.utils.logger import get_logger
+from .backbones import build_resnet_backbone
+from .fastinst.fastinst_decoder import FastInstDecoder
+from .fastinst.fastinst_encoder import PyramidPoolingModuleFPN
+
+logger = get_logger()
+
+
+@MODELS.register_module(Tasks.image_segmentation, module_name=Models.fastinst)
+class FastInst(TorchModel):
+
+    def __init__(self,
+                 model_dir,
+                 backbone=None,
+                 encoder=None,
+                 decoder=None,
+                 pretrained=None,
+                 classes=None,
+                 **kwargs):
+        """
+        Deep Learning Technique for Human Parsing: A Survey and Outlook. See https://arxiv.org/abs/2301.00394
+        Args:
+            backbone (dict): backbone config.
+            encoder (dict): encoder config.
+            decoder (dict): decoder config.
+            pretrained (bool): whether to use pretrained model
+            classes (list): class names
+        """
+        super(FastInst, self).__init__(model_dir, **kwargs)
+
+        self.backbone = build_resnet_backbone(
+            **backbone, input_shape={'channels': 3})
+        in_features = encoder.pop('in_features')
+        input_shape = {
+            k: v
+            for k, v in self.backbone.output_shape().items()
+            if k in in_features
+        }
+        encoder = PyramidPoolingModuleFPN(input_shape=input_shape, **encoder)
+        decoder = FastInstDecoder(in_channels=encoder.convs_dim, **decoder)
+        self.sem_seg_head = FastInstHead(
+            pixel_decoder=encoder, transformer_predictor=decoder)
+
+        self.num_classes = decoder.num_classes
+        self.num_queries = decoder.num_queries
+        self.size_divisibility = 32
+        self.register_buffer(
+            'pixel_mean',
+            torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1), False)
+        self.register_buffer(
+            'pixel_std',
+            torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1), False)
+        self.classes = classes
+        self.test_topk_per_image = 100
+
+        if pretrained:
+            model_path = os.path.join(model_dir, ModelFile.TORCH_MODEL_FILE)
+            logger.info(f'loading model from {model_path}')
+            weight = torch.load(model_path, map_location='cpu')['model']
+            tgt_weight = self.state_dict()
+            for name in list(weight.keys()):
+                if name in tgt_weight:
+                    load_size = weight[name].size()
+                    tgt_size = tgt_weight[name].size()
+                    mis_match = False
+                    if len(load_size) != len(tgt_size):
+                        mis_match = True
+                    else:
+                        for n1, n2 in zip(load_size, tgt_size):
+                            if n1 != n2:
+                                mis_match = True
+                                break
+                    if mis_match:
+                        logger.info(
+                            f'size mismatch for {name} '
+                            f'({load_size} -> {tgt_size}), skip loading.')
+                        del weight[name]
+                else:
+                    logger.info(
+                        f'{name} doesn\'t exist in current model, skip loading.'
+                    )
+
+            self.load_state_dict(weight, strict=False)
+            logger.info('load model done')
+
+    def forward(self, batched_inputs: List[dict]) -> Dict[str, Any]:
+        images = [x['image'].to(self.device) for x in batched_inputs]
+        images = [(x - self.pixel_mean) / self.pixel_std for x in images]
+        images = ImageList.from_tensors(images, self.size_divisibility)
+
+        features = self.backbone(images.tensor)
+        outputs = self.sem_seg_head(features)
+
+        return dict(
+            outputs=outputs, batched_inputs=batched_inputs, images=images)
+
+    def postprocess(self, input: Dict[str, Any]) -> Dict[str, Any]:
+        outputs = input['outputs']
+        batched_inputs = input['batched_inputs']
+        images = input['images']
+        if self.training:
+            raise NotImplementedError
+        else:
+            mask_cls_results = outputs['pred_logits']  # (B, Q, C+1)
+            mask_pred_results = outputs['pred_masks']  # (B, Q, H, W)
+            # upsample masks
+            mask_pred_results = F.interpolate(
+                mask_pred_results,
+                size=(images.tensor.shape[-2], images.tensor.shape[-1]),
+                mode='bilinear',
+                align_corners=False,
+            )
+
+            del outputs
+
+            processed_results = []
+            for mask_cls_result, mask_pred_result, input_per_image, image_size in zip(
+                    mask_cls_results, mask_pred_results, batched_inputs,
+                    images.image_sizes):
+                height = input_per_image.get('height', image_size[0])
+                width = input_per_image.get('width', image_size[1])
+                processed_results.append({})  # for each image
+
+                mask_pred_result = self.sem_seg_postprocess(
+                    mask_pred_result, image_size, height, width)
+                mask_cls_result = mask_cls_result.to(mask_pred_result)
+
+                instance_r = self.instance_inference(mask_cls_result,
+                                                     mask_pred_result)
+                processed_results[-1]['instances'] = instance_r
+
+        return dict(eval_result=processed_results)
+
+    @property
+    def device(self):
+        return self.pixel_mean.device
+
+    def sem_seg_postprocess(self, result, img_size, output_height,
+                            output_width):
+        result = result[:, :img_size[0], :img_size[1]].expand(1, -1, -1, -1)
+        result = F.interpolate(
+            result,
+            size=(output_height, output_width),
+            mode='bilinear',
+            align_corners=False)[0]
+        return result
+
+    def instance_inference(self, mask_cls, mask_pred):
+        # mask_pred is already processed to have the same shape as original input
+        image_size = mask_pred.shape[-2:]
+
+        # [Q, K]
+        scores = F.softmax(mask_cls, dim=-1)[:, :-1]
+        labels = torch.arange(
+            self.num_classes,
+            device=self.device).unsqueeze(0).repeat(self.num_queries,
+                                                    1).flatten(0, 1)
+        scores_per_image, topk_indices = scores.flatten(0, 1).topk(
+            self.test_topk_per_image, sorted=False)
+        labels_per_image = labels[topk_indices]
+
+        topk_indices = topk_indices // self.num_classes
+        mask_pred = mask_pred[topk_indices]
+
+        result = {'image_size': image_size}
+        # mask (before sigmoid)
+        mask_pred_sigmoid = mask_pred.sigmoid()
+        result['pred_masks'] = (mask_pred_sigmoid > 0.5).float()
+
+        # calculate average mask prob
+        mask_scores_per_image = (mask_pred_sigmoid.flatten(1)
+                                 * result['pred_masks'].flatten(1)).sum(1) / (
+                                     result['pred_masks'].flatten(1).sum(1)
+                                     + 1e-6)
+        result['scores'] = scores_per_image * mask_scores_per_image
+        result['pred_classes'] = labels_per_image
+        return result
+
+
+class FastInstHead(nn.Module):
+
+    def __init__(
+            self,
+            *,
+            pixel_decoder: nn.Module,
+            # extra parameters
+            transformer_predictor: nn.Module):
+        """
+        NOTE: this interface is experimental.
+        Args:
+            pixel_decoder: the pixel decoder module
+            transformer_predictor: the transformer decoder that makes prediction
+        """
+        super().__init__()
+        self.pixel_decoder = pixel_decoder
+        self.predictor = transformer_predictor
+
+    def forward(self, features, targets=None):
+        return self.layers(features, targets)
+
+    def layers(self, features, targets=None):
+        mask_features, multi_scale_features = self.pixel_decoder.forward_features(
+            features)
+        predictions = self.predictor(multi_scale_features, mask_features,
+                                     targets)
+        return predictions

modelscope/pipelines/audio/speaker_change_locating_pipeline.py

@@ -0,0 +1,105 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import io
+from typing import Any, Dict, List, Union
+
+import numpy as np
+import soundfile as sf
+import torch
+
+from modelscope.fileio import File
+from modelscope.metainfo import Pipelines
+from modelscope.outputs import OutputKeys
+from modelscope.pipelines.base import InputModel, Pipeline
+from modelscope.pipelines.builder import PIPELINES
+from modelscope.utils.constant import Tasks
+from modelscope.utils.logger import get_logger
+
+logger = get_logger()
+
+__all__ = ['SpeakerChangeLocatingPipeline']
+
+
+@PIPELINES.register_module(
+    Tasks.speaker_diarization, module_name=Pipelines.speaker_change_locating)
+class SpeakerChangeLocatingPipeline(Pipeline):
+    """Speaker Change Locating Inference Pipeline
+    use `model` to create a speaker change Locating pipeline.
+
+    Args:
+        model (SpeakerChangeLocatingPipeline): A model instance, or a model local dir, or a model id in the model hub.
+        kwargs (dict, `optional`):
+            Extra kwargs passed into the pipeline's constructor.
+    Example:
+    >>> from modelscope.pipelines import pipeline
+    >>> from modelscope.utils.constant import Tasks
+    >>> p = pipeline(
+    >>>    task=Tasks.speaker_diarization, model='damo/speech_campplus-transformer_scl_zh-cn_16k-common')
+    >>> print(p(audio))
+
+    """
+
+    def __init__(self, model: InputModel, **kwargs):
+        """use `model` to create a speaker change Locating pipeline for prediction
+        Args:
+            model (str): a valid offical model id
+        """
+        super().__init__(model=model, **kwargs)
+        self.model_config = self.model.model_config
+        self.config = self.model.model_config
+        self.anchor_size = self.config['anchor_size']
+
+    def __call__(self, audio: str, embds: List = None) -> Dict[str, Any]:
+        if embds is not None:
+            assert len(embds) == 2
+            assert isinstance(embds[0], np.ndarray) and isinstance(
+                embds[1], np.ndarray)
+            assert embds[0].shape == (
+                self.anchor_size, ) and embds[1].shape == (self.anchor_size, )
+        else:
+            embd1 = np.zeros(self.anchor_size // 2)
+            embd2 = np.ones(self.anchor_size - self.anchor_size // 2)
+            embd3 = np.ones(self.anchor_size // 2)
+            embd4 = np.zeros(self.anchor_size - self.anchor_size // 2)
+            embds = [
+                np.stack([embd1, embd2], axis=1).flatten(),
+                np.stack([embd3, embd4], axis=1).flatten(),
+            ]
+        anchors = torch.from_numpy(np.stack(embds,
+                                            axis=0)).float().unsqueeze(0)
+
+        output = self.preprocess(audio)
+        output = self.forward(output, anchors)
+        output = self.postprocess(output)
+
+        return output
+
+    def forward(self, input: torch.Tensor, anchors: torch.Tensor):
+        output = self.model(input, anchors)
+        return output
+
+    def postprocess(self, input: torch.Tensor) -> Dict[str, Any]:
+        predict = np.where(np.diff(input.argmax(-1).numpy()))
+        try:
+            predict = predict[0][0] * 0.01 + 0.02
+            predict = round(predict, 2)
+            return {OutputKeys.TEXT: f'The change point is at {predict}s.'}
+        except Exception:
+            return {OutputKeys.TEXT: 'No change point is found.'}
+
+    def preprocess(self, input: str) -> torch.Tensor:
+        if isinstance(input, str):
+            file_bytes = File.read(input)
+            data, fs = sf.read(io.BytesIO(file_bytes), dtype='float32')
+            if len(data.shape) == 2:
+                data = data[:, 0]
+            if fs != self.model_config['sample_rate']:
+                raise ValueError(
+                    'modelscope error: Only support %d sample rate files'
+                    % self.model_cfg['sample_rate'])
+            data = torch.from_numpy(data).unsqueeze(0)
+        else:
+            raise ValueError(
+                'modelscope error: The input type is restricted to audio file address'
+                % i)
+        return data

modelscope/pipelines/cv/fast_instance_segmentation_pipeline.py

@@ -0,0 +1,116 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import Any, Dict, Optional, Union
+
+import numpy as np
+import torch
+import torchvision.transforms as T
+
+from modelscope.metainfo import Pipelines
+from modelscope.models.cv.image_instance_segmentation import FastInst
+from modelscope.outputs import OutputKeys
+from modelscope.pipelines.base import Input, Pipeline
+from modelscope.pipelines.builder import PIPELINES
+from modelscope.preprocessors import LoadImage
+from modelscope.utils.constant import Tasks
+from modelscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@PIPELINES.register_module(
+    Tasks.image_segmentation, module_name=Pipelines.fast_instance_segmentation)
+class FastInstanceSegmentationPipeline(Pipeline):
+
+    def __init__(self,
+                 model: Union[FastInst, str],
+                 preprocessor: Optional = None,
+                 **kwargs):
+        r"""The inference pipeline for fastinst models.
+
+        The model outputs a dict with keys of `scores`, `labels`, and `masks`.
+
+        Args:
+            model (`str` or `Model` or module instance): A model instance or a model local dir
+                or a model id in the model hub.
+            preprocessor (`Preprocessor`, `optional`): A Preprocessor instance.
+            kwargs (dict, `optional`):
+                Extra kwargs passed into the preprocessor's constructor.
+
+        Examples:
+            >>> from modelscope.outputs import OutputKeys
+            >>> from modelscope.pipelines import pipeline
+            >>> pipeline_ins = pipeline('image-segmentation',
+                model='damo/cv_resnet50_fast-instance-segmentation_coco')
+            >>> input_img = 'https://modelscope.oss-cn-beijing.aliyuncs.com/test/images/image_instance_segmentation.jpg'
+            >>> print(pipeline_ins(input_img)[OutputKeys.LABELS])
+        """
+        super().__init__(model=model, preprocessor=preprocessor, **kwargs)
+        self.model.eval()
+
+    def _get_preprocess_shape(self, oldh, oldw, short_edge_length, max_size):
+        h, w = oldh, oldw
+        size = short_edge_length * 1.0
+        scale = size / min(h, w)
+        if h < w:
+            newh, neww = size, scale * w
+        else:
+            newh, neww = scale * h, size
+        if max(newh, neww) > max_size:
+            scale = max_size * 1.0 / max(newh, neww)
+            newh = newh * scale
+            neww = neww * scale
+        neww = int(neww + 0.5)
+        newh = int(newh + 0.5)
+        return (newh, neww)
+
+    def preprocess(self,
+                   input: Input,
+                   min_size=640,
+                   max_size=1333) -> Dict[str, Any]:
+        image = LoadImage.convert_to_img(input)
+        w, h = image.size[:2]
+        dataset_dict = {'width': w, 'height': h}
+        new_h, new_w = self._get_preprocess_shape(h, w, min_size, max_size)
+        test_transforms = T.Compose([
+            T.Resize((new_h, new_w)),
+            T.ToTensor(),
+        ])
+        image = test_transforms(image)
+        dataset_dict['image'] = image * 255.
+        result = {'batched_inputs': [dataset_dict]}
+        return result
+
+    def forward(self, input: Dict[str, Any],
+                **forward_params) -> Dict[str, Any]:
+        with torch.no_grad():
+            output = self.model(**input)
+        return output
+
+    def postprocess(self,
+                    inputs: Dict[str, Any],
+                    score_thr=0.5) -> Dict[str, Any]:
+        predictions = inputs['eval_result'][0]['instances']
+        scores = predictions['scores'].detach().cpu().numpy()
+        pred_masks = predictions['pred_masks'].detach().cpu().numpy()
+        pred_classes = predictions['pred_classes'].detach().cpu().numpy()
+
+        thresholded_idxs = np.array(scores) >= score_thr
+        scores = scores[thresholded_idxs]
+        pred_classes = pred_classes[thresholded_idxs]
+        pred_masks = pred_masks[thresholded_idxs]
+
+        results_dict = {
+            OutputKeys.MASKS: [],
+            OutputKeys.LABELS: [],
+            OutputKeys.SCORES: []
+        }
+        for score, cls, mask in zip(scores, pred_classes, pred_masks):
+            score = np.float64(score)
+            label = self.model.classes[int(cls)]
+            mask = np.array(mask, dtype=np.float64)
+
+            results_dict[OutputKeys.SCORES].append(score)
+            results_dict[OutputKeys.LABELS].append(label)
+            results_dict[OutputKeys.MASKS].append(mask)
+
+        return results_dict

tests/pipelines/test_fast_instance_segmentation.py

@@ -0,0 +1,39 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import unittest
+
+from modelscope.models import Model
+from modelscope.outputs import OutputKeys
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+from modelscope.utils.demo_utils import DemoCompatibilityCheck
+from modelscope.utils.test_utils import test_level
+
+
+class FastInstanceSegmentationTest(unittest.TestCase, DemoCompatibilityCheck):
+
+    def setUp(self) -> None:
+        self.task = Tasks.image_segmentation
+        self.model_id = 'damo/cv_resnet50_fast-instance-segmentation_coco'
+
+    image = 'data/test/images/image_instance_segmentation.jpg'
+
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_run_with_model_name(self):
+        pipeline_parsing = pipeline(
+            task=Tasks.image_segmentation, model=self.model_id)
+        print(pipeline_parsing(input=self.image)[OutputKeys.LABELS])
+
+    @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
+    def test_run_with_model_from_modelhub(self):
+        model = Model.from_pretrained(self.model_id)
+        pipeline_parsing = pipeline(
+            task=Tasks.image_segmentation, model=model, preprocessor=None)
+        print(pipeline_parsing(input=self.image)[OutputKeys.LABELS])
+
+    @unittest.skip('demo compatibility test is only enabled on a needed-basis')
+    def test_demo_compatibility(self):
+        self.compatibility_check()
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/pipelines/test_speaker_verification.py

@@ -2,7 +2,7 @@
 
 import os.path
 import unittest
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Union
 
 from modelscope.outputs import OutputKeys
 from modelscope.pipelines import pipeline
@@ -16,20 +16,25 @@ logger = get_logger()
 SPEAKER1_A_EN_16K_WAV = 'data/test/audios/speaker1_a_en_16k.wav'
 SPEAKER1_B_EN_16K_WAV = 'data/test/audios/speaker1_b_en_16k.wav'
 SPEAKER2_A_EN_16K_WAV = 'data/test/audios/speaker2_a_en_16k.wav'
+SCL_EXAMPLE_WAV = 'data/test/audios/scl_example1.wav'
 
 
 class SpeakerVerificationTest(unittest.TestCase, DemoCompatibilityCheck):
     ecapatdnn_voxceleb_16k_model_id = 'damo/speech_ecapa-tdnn_sv_en_voxceleb_16k'
     campplus_voxceleb_16k_model_id = 'damo/speech_campplus_sv_en_voxceleb_16k'
     rdino_voxceleb_16k_model_id = 'damo/speech_rdino_ecapa_tdnn_sv_en_voxceleb_16k'
+    speaker_change_locating_cn_model_id = 'damo/speech_campplus-transformer_scl_zh-cn_16k-common'
 
     def setUp(self) -> None:
         self.task = Tasks.speaker_verification
 
     def run_pipeline(self,
                      model_id: str,
-                     audios: List[str],
+                     audios: Union[List[str], str],
+                     task: str = None,
                      model_revision=None) -> Dict[str, Any]:
+        if task is not None:
+            self.task = task
         p = pipeline(
             task=self.task, model=model_id, model_revision=model_revision)
         result = p(audios)
@@ -66,6 +71,17 @@ class SpeakerVerificationTest(unittest.TestCase, DemoCompatibilityCheck):
         print(result)
         self.assertTrue(OutputKeys.SCORE in result)
 
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
+    def test_run_with_speaker_change_locating_cn_16k(self):
+        logger.info(
+            'Run speaker change locating for campplus-transformer model')
+        result = self.run_pipeline(
+            model_id=self.speaker_change_locating_cn_model_id,
+            task=Tasks.speaker_diarization,
+            audios=SCL_EXAMPLE_WAV)
+        print(result)
+        self.assertTrue(OutputKeys.TEXT in result)
+
     @unittest.skip('demo compatibility test is only enabled on a needed-basis')
     def test_demo_compatibility(self):
         self.compatibility_check()