diff --git a/modelscope/metainfo.py b/modelscope/metainfo.py index f07192ea..efce05b0 100644 --- a/modelscope/metainfo.py +++ b/modelscope/metainfo.py @@ -116,6 +116,7 @@ class Models(object): bad_image_detecting = 'bad-image-detecting' controllable_image_generation = 'controllable-image-generation' longshortnet = 'longshortnet' + fastinst = 'fastinst' pedestrian_attribute_recognition = 'pedestrian-attribute-recognition' # nlp models @@ -181,6 +182,7 @@ class Models(object): generic_sv = 'generic-sv' ecapa_tdnn_sv = 'ecapa-tdnn-sv' campplus_sv = 'cam++-sv' + scl_sd = 'scl-sd' rdino_tdnn_sv = 'rdino_ecapa-tdnn-sv' generic_lm = 'generic-lm' @@ -396,7 +398,7 @@ class Pipelines(object): nerf_recon_acc = 'nerf-recon-acc' bad_image_detecting = 'bad-image-detecting' controllable_image_generation = 'controllable-image-generation' - + fast_instance_segmentation = 'fast-instance-segmentation' image_quality_assessment_mos = 'image-quality-assessment-mos' image_quality_assessment_man = 'image-quality-assessment-man' image_quality_assessment_degradation = 'image-quality-assessment-degradation' @@ -480,6 +482,7 @@ class Pipelines(object): vad_inference = 'vad-inference' speaker_verification = 'speaker-verification' speaker_verification_rdino = 'speaker-verification-rdino' + speaker_change_locating = 'speaker-change-locating' lm_inference = 'language-score-prediction' speech_timestamp_inference = 'speech-timestamp-inference' diff --git a/modelscope/models/audio/sv/DTDNN.py b/modelscope/models/audio/sv/DTDNN.py index d9e21ce8..d86d6799 100644 --- a/modelscope/models/audio/sv/DTDNN.py +++ b/modelscope/models/audio/sv/DTDNN.py @@ -76,11 +76,13 @@ class CAMPPlus(nn.Module): bn_size=4, init_channels=128, config_str='batchnorm-relu', - memory_efficient=True): + memory_efficient=True, + output_level='segment'): super(CAMPPlus, self).__init__() self.head = FCM(feat_dim=feat_dim) channels = self.head.out_channels + self.output_level = output_level self.xvector = nn.Sequential( OrderedDict([ @@ -118,10 +120,14 @@ class CAMPPlus(nn.Module): self.xvector.add_module('out_nonlinear', get_nonlinear(config_str, channels)) - self.xvector.add_module('stats', StatsPool()) - self.xvector.add_module( - 'dense', - DenseLayer(channels * 2, embedding_size, config_str='batchnorm_')) + if self.output_level == 'segment': + self.xvector.add_module('stats', StatsPool()) + self.xvector.add_module( + 'dense', + DenseLayer( + channels * 2, embedding_size, config_str='batchnorm_')) + else: + assert self.output_level == 'frame', '`output_level` should be set to \'segment\' or \'frame\'. ' for m in self.modules(): if isinstance(m, (nn.Conv1d, nn.Linear)): @@ -133,6 +139,8 @@ class CAMPPlus(nn.Module): x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T) x = self.head(x) x = self.xvector(x) + if self.output_level == 'frame': + x = x.transpose(1, 2) return x diff --git a/modelscope/models/audio/sv/speaker_change_locator.py b/modelscope/models/audio/sv/speaker_change_locator.py new file mode 100644 index 00000000..c22e4c1b --- /dev/null +++ b/modelscope/models/audio/sv/speaker_change_locator.py @@ -0,0 +1,319 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from collections import OrderedDict +from typing import Any, Dict, Union + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchaudio.compliance.kaldi as Kaldi + +from modelscope.metainfo import Models +from modelscope.models import MODELS, TorchModel +from modelscope.models.audio.sv.DTDNN import CAMPPlus +from modelscope.utils.constant import Tasks + + +class MultiHeadSelfAttention(nn.Module): + + def __init__(self, n_units, h=8, dropout=0.1): + super(MultiHeadSelfAttention, self).__init__() + self.linearQ = nn.Linear(n_units, n_units) + self.linearK = nn.Linear(n_units, n_units) + self.linearV = nn.Linear(n_units, n_units) + self.linearO = nn.Linear(n_units, n_units) + self.d_k = n_units // h + self.h = h + self.dropout = nn.Dropout(p=dropout) + self.att = None + + def forward(self, x, batch_size): + # x: (BT, F) + q = self.linearQ(x).reshape(batch_size, -1, self.h, self.d_k) + k = self.linearK(x).reshape(batch_size, -1, self.h, self.d_k) + v = self.linearV(x).reshape(batch_size, -1, self.h, self.d_k) + scores = torch.matmul(q.transpose(1, 2), k.permute( + 0, 2, 3, 1)) / np.sqrt(self.d_k) + # scores: (B, h, T, T) + self.att = F.softmax(scores, dim=3) + p_att = self.dropout(self.att) + # v : (B, T, h, d_k) + # p_att : (B, h, T, T) + x = torch.matmul(p_att, v.transpose(1, 2)) + # x : (B, h, T, d_k) + x = x.transpose(1, 2).reshape(-1, self.h * self.d_k) + return self.linearO(x) + + +class PositionwiseFeedForward(nn.Module): + + def __init__(self, n_units, d_units, dropout): + super(PositionwiseFeedForward, self).__init__() + self.linear1 = nn.Linear(n_units, d_units) + self.linear2 = nn.Linear(d_units, n_units) + self.dropout = nn.Dropout(p=dropout) + + def forward(self, x): + return self.linear2(self.dropout(F.relu(self.linear1(x)))) + + +class PosEncoding(nn.Module): + + def __init__(self, max_seq_len, d_word_vec): + super(PosEncoding, self).__init__() + pos_enc = np.array([[ + pos / np.power(10000, 2.0 * (j // 2) / d_word_vec) + for j in range(d_word_vec) + ] for pos in range(max_seq_len)]) + pos_enc[:, 0::2] = np.sin(pos_enc[:, 0::2]) + pos_enc[:, 1::2] = np.cos(pos_enc[:, 1::2]) + pad_row = np.zeros([1, d_word_vec]) + pos_enc = np.concatenate([pad_row, pos_enc]).astype(np.float32) + + self.pos_enc = torch.nn.Embedding(max_seq_len + 1, d_word_vec) + self.pos_enc.weight = torch.nn.Parameter( + torch.from_numpy(pos_enc), requires_grad=False) + + def forward(self, input_len): + max_len = torch.max(input_len) + input_pos = torch.LongTensor([ + list(range(1, len + 1)) + [0] * (max_len - len) + for len in input_len + ]) + + return self.pos_enc(input_pos) + + +class TransformerEncoder(nn.Module): + + def __init__(self, + idim, + n_units=256, + n_layers=2, + e_units=512, + h=4, + dropout=0.1): + super(TransformerEncoder, self).__init__() + self.linear_in = nn.Linear(idim, n_units) + self.lnorm_in = nn.LayerNorm(n_units) + + self.n_layers = n_layers + self.dropout = nn.Dropout(p=dropout) + for i in range(n_layers): + setattr(self, '{}{:d}'.format('lnorm1_', i), nn.LayerNorm(n_units)) + setattr(self, '{}{:d}'.format('self_att_', i), + MultiHeadSelfAttention(n_units, h)) + setattr(self, '{}{:d}'.format('lnorm2_', i), nn.LayerNorm(n_units)) + setattr(self, '{}{:d}'.format('ff_', i), + PositionwiseFeedForward(n_units, e_units, dropout)) + self.lnorm_out = nn.LayerNorm(n_units) + + def forward(self, x): + # x: [B, num_anchors, T, n_in] + bs, num, tframe, dim = x.size() + x = x.reshape(bs * num, tframe, -1) # [B*num_anchors, T, dim] + # x: (B, T, F) ... batch, time, (mel)freq + B_size, T_size, _ = x.shape + # e: (BT, F) + e = self.linear_in(x.reshape(B_size * T_size, -1)) + # Encoder stack + for i in range(self.n_layers): + # layer normalization + e = getattr(self, '{}{:d}'.format('lnorm1_', i))(e) + # self-attention + s = getattr(self, '{}{:d}'.format('self_att_', i))(e, x.shape[0]) + # residual + e = e + self.dropout(s) + # layer normalization + e = getattr(self, '{}{:d}'.format('lnorm2_', i))(e) + # positionwise feed-forward + s = getattr(self, '{}{:d}'.format('ff_', i))(e) + # residual + e = e + self.dropout(s) + # final layer normalization + # output: (BT, F) + # output: (B, F, T) + output = self.lnorm_out(e).reshape(B_size, T_size, -1) + output = output.reshape(bs, num, tframe, + -1) # [B, num_anchors, T, dim] + return output + + +class TransformerEncoder_out(nn.Module): + + def __init__(self, + idim, + n_units=256, + n_layers=2, + e_units=512, + h=4, + dropout=0.1): + super(TransformerEncoder_out, self).__init__() + self.linear_in = nn.Linear(idim, n_units) + self.lnorm_in = nn.LayerNorm(n_units) + + self.n_layers = n_layers + self.dropout = nn.Dropout(p=dropout) + for i in range(n_layers): + setattr(self, '{}{:d}'.format('lnorm1_', i), nn.LayerNorm(n_units)) + setattr(self, '{}{:d}'.format('self_att_', i), + MultiHeadSelfAttention(n_units, h)) + setattr(self, '{}{:d}'.format('lnorm2_', i), nn.LayerNorm(n_units)) + setattr(self, '{}{:d}'.format('ff_', i), + PositionwiseFeedForward(n_units, e_units, dropout)) + self.lnorm_out = nn.LayerNorm(n_units) + + def forward(self, x): + # x: (B, T, F) + B_size, T_size, _ = x.shape + # e: (BT, F) + e = self.linear_in(x.reshape(B_size * T_size, -1)) + # Encoder stack + for i in range(self.n_layers): + # layer normalization + e = getattr(self, '{}{:d}'.format('lnorm1_', i))(e) + # self-attention + s = getattr(self, '{}{:d}'.format('self_att_', i))(e, x.shape[0]) + # residual + e = e + self.dropout(s) + # layer normalization + e = getattr(self, '{}{:d}'.format('lnorm2_', i))(e) + # positionwise feed-forward + s = getattr(self, '{}{:d}'.format('ff_', i))(e) + # residual + e = e + self.dropout(s) + # final layer normalization + # output: (BT, F) + # output: (B, T, F) + output = self.lnorm_out(e).reshape(B_size, T_size, -1) + return output + + +class OutLayer(nn.Module): + + def __init__(self, n_units=256, num_anchors=2): + super(OutLayer, self).__init__() + self.combine = TransformerEncoder_out(num_anchors * n_units, n_units) + self.out_linear = nn.Linear(n_units // num_anchors, 1) + + def forward(self, input): + # input: [B, num_anchors, T, dim] + bs, num, tframe, dim = input.size() + output = input.permute(0, 2, 1, + 3).reshape(bs, tframe, + -1) # [Bs, t, num_anchors*dim] + output = self.combine(output) # [Bs, t, n_units] + output = output.reshape( + bs, tframe, num, -1) # [Bs, t, num_anchors, n_units//num_anchors] + output = self.out_linear(output).squeeze(-1) # [Bs, t, num_anchors] + + return output + + +class TransformerDetector(nn.Module): + + def __init__(self, + frame_dim=512, + anchor_dim=192, + hidden_dim=256, + max_seq_len=1000): + super(TransformerDetector, self).__init__() + self.detection = TransformerEncoder( + idim=frame_dim + anchor_dim, n_units=hidden_dim) + self.output = OutLayer(n_units=hidden_dim) + self.pos_enc = PosEncoding(max_seq_len, hidden_dim) + + def forward(self, feats, anchors): + # feats: [1, t, fdim] + num_frames = feats.shape[1] + num_anchors = anchors.shape[1] + bs = feats.shape[0] + feats = feats.unsqueeze(1).repeat( + 1, num_anchors, 1, 1) # shape: [Bs, num_anchors, t, fdim] + anchors = anchors.unsqueeze(2).repeat( + 1, 1, num_frames, 1) # shape: [Bs, num_anchors, t, xdim] + sd_in = torch.cat((feats, anchors), + dim=-1) # shape: [Bs, num_anchors, t, fdim+xdim] + sd_out = self.detection(sd_in) # shape: [Bs, num_anchors, t, sd_dim] + + # pos + pos_emb = self.pos_enc(torch.tensor([num_frames] * (bs * num_anchors))) + pos_emb = pos_emb.reshape(bs, num_anchors, num_frames, -1) + sd_out += pos_emb + + # output + output = self.output(sd_out) # shape: [Bs, t, num_anchors] + + return output + + +@MODELS.register_module(Tasks.speaker_diarization, module_name=Models.scl_sd) +class SpeakerChangeLocatorTransformer(TorchModel): + r"""A speaekr change locator using the transformer architecture as the backbone. + Args: + model_dir: A model dir. + model_config: The model config. + """ + + def __init__(self, model_dir, model_config: Dict[str, Any], *args, + **kwargs): + super().__init__(model_dir, model_config, *args, **kwargs) + self.model_config = model_config + + self.feature_dim = self.model_config['fbank_dim'] + frame_size = self.model_config['frame_size'] + anchor_size = self.model_config['anchor_size'] + + self.encoder = CAMPPlus(self.feature_dim, output_level='frame') + self.backend = TransformerDetector( + frame_dim=frame_size, anchor_dim=anchor_size) + + pretrained_encoder = kwargs['pretrained_encoder'] + pretrained_backend = kwargs['pretrained_backend'] + + self.__load_check_point(pretrained_encoder, pretrained_backend) + + self.encoder.eval() + self.backend.eval() + + def forward(self, audio, anchors): + assert len(audio.shape) == 2 and audio.shape[ + 0] == 1, 'modelscope error: the shape of input audio to model needs to be [1, T]' + assert len( + anchors.shape + ) == 3 and anchors.shape[0] == 1 and anchors.shape[ + 1] == 2, 'modelscope error: the shape of input anchors to model needs to be [1, 2, D]' + # audio shape: [1, T] + feature = self.__extract_feature(audio) + frame_state = self.encoder(feature) + output = self.backend(frame_state, anchors) + output = output.squeeze(0).detach().cpu().sigmoid() + + time_scale_factor = int(np.ceil(feature.shape[1] / output.shape[0])) + output = output.unsqueeze(1).expand(-1, time_scale_factor, + -1).reshape(-1, output.shape[-1]) + return output + + def __extract_feature(self, audio): + feature = Kaldi.fbank(audio, num_mel_bins=self.feature_dim) + feature = feature - feature.mean(dim=0, keepdim=True) + feature = feature.unsqueeze(0) + return feature + + def __load_check_point(self, + pretrained_encoder, + pretrained_backend, + device=None): + if not device: + device = torch.device('cpu') + self.encoder.load_state_dict( + torch.load( + os.path.join(self.model_dir, pretrained_encoder), + map_location=device)) + + self.backend.load_state_dict( + torch.load( + os.path.join(self.model_dir, pretrained_backend), + map_location=device)) diff --git a/modelscope/models/cv/image_instance_segmentation/__init__.py b/modelscope/models/cv/image_instance_segmentation/__init__.py index 60e688eb..8041a7e7 100644 --- a/modelscope/models/cv/image_instance_segmentation/__init__.py +++ b/modelscope/models/cv/image_instance_segmentation/__init__.py @@ -8,10 +8,12 @@ if TYPE_CHECKING: from .maskdino_swin import MaskDINOSwin from .model import CascadeMaskRCNNSwinModel from .maskdino_model import MaskDINOSwinModel + from .fastinst_model import FastInst from .postprocess_utils import get_img_ins_seg_result, get_maskdino_ins_seg_result else: _import_structure = { 'cascade_mask_rcnn_swin': ['CascadeMaskRCNNSwin'], + 'fastinst_model': ['FastInst'], 'maskdino_swin': ['MaskDINOSwin'], 'model': ['CascadeMaskRCNNSwinModel'], 'maskdino_model': ['MaskDINOSwinModel'], diff --git a/modelscope/models/cv/image_instance_segmentation/backbones/__init__.py b/modelscope/models/cv/image_instance_segmentation/backbones/__init__.py index bbeac51e..1e7325f3 100644 --- a/modelscope/models/cv/image_instance_segmentation/backbones/__init__.py +++ b/modelscope/models/cv/image_instance_segmentation/backbones/__init__.py @@ -6,10 +6,12 @@ from modelscope.utils.import_utils import LazyImportModule if TYPE_CHECKING: from .swin_transformer import SwinTransformer from .swin_transformer import D2SwinTransformer + from .resnet import build_resnet_backbone else: _import_structure = { 'swin_transformer': ['SwinTransformer', 'D2SwinTransformer'], + 'resnet': ['build_resnet_backbone'] } import sys diff --git a/modelscope/models/cv/image_instance_segmentation/backbones/resnet.py b/modelscope/models/cv/image_instance_segmentation/backbones/resnet.py new file mode 100644 index 00000000..4e2a5ec1 --- /dev/null +++ b/modelscope/models/cv/image_instance_segmentation/backbones/resnet.py @@ -0,0 +1,114 @@ +# Part of the implementation is borrowed and modified from Detectron2, publicly available at +# https://github.com/facebookresearch/detectron2/blob/main/projects/DeepLab/deeplab/resnet.py + +import torch.nn.functional as F +from torch import nn + +from modelscope.models.cv.image_human_parsing.backbone.deeplab_resnet import ( + BottleneckBlock, DeeplabResNet, get_norm) +from modelscope.models.cv.image_instance_segmentation.maskdino.utils import \ + Conv2d + + +class BasicStem(nn.Module): + """ + The standard ResNet stem (layers before the first residual block), + with a conv, relu and max_pool. + """ + + def __init__(self, in_channels=3, out_channels=64, norm='BN'): + """ + Args: + norm (str or callable): norm after the first conv layer. + See :func:`layers.get_norm` for supported format. + """ + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.stride = 4 + self.conv1 = Conv2d( + in_channels, + out_channels, + kernel_size=7, + stride=2, + padding=3, + bias=False, + norm=get_norm(norm, out_channels), + ) + + def forward(self, x): + x = self.conv1(x) + x = F.relu_(x) + x = F.max_pool2d(x, kernel_size=3, stride=2, padding=1) + return x + + +def build_resnet_backbone(out_features, depth, num_groups, width_per_group, + norm, stem_out_channels, res2_out_channels, + stride_in_1x1, res4_dilation, res5_dilation, + res5_multi_grid, input_shape): + stem = BasicStem( + in_channels=input_shape['channels'], + out_channels=stem_out_channels, + norm=norm) + bottleneck_channels = num_groups * width_per_group + in_channels = stem_out_channels + out_channels = res2_out_channels + + assert res4_dilation in { + 1, 2 + }, 'res4_dilation cannot be {}.'.format(res4_dilation) + assert res5_dilation in { + 1, 2, 4 + }, 'res5_dilation cannot be {}.'.format(res5_dilation) + if res4_dilation == 2: + # Always dilate res5 if res4 is dilated. + assert res5_dilation == 4 + + num_blocks_per_stage = { + 50: [3, 4, 6, 3], + 101: [3, 4, 23, 3], + 152: [3, 8, 36, 3] + }[depth] + + stages = [] + out_stage_idx = [{ + 'res2': 2, + 'res3': 3, + 'res4': 4, + 'res5': 5 + }[f] for f in out_features] + max_stage_idx = max(out_stage_idx) + for idx, stage_idx in enumerate(range(2, max_stage_idx + 1)): + if stage_idx == 4: + dilation = res4_dilation + elif stage_idx == 5: + dilation = res5_dilation + else: + dilation = 1 + first_stride = 1 if idx == 0 or dilation > 1 else 2 + stride_per_block = [first_stride] + stride_per_block += [1] * (num_blocks_per_stage[idx] - 1) + stage_kargs = { + 'num_blocks': num_blocks_per_stage[idx], + 'stride_per_block': stride_per_block, + 'in_channels': in_channels, + 'out_channels': out_channels, + 'norm': norm, + 'bottleneck_channels': bottleneck_channels, + 'stride_in_1x1': stride_in_1x1, + 'dilation': dilation, + 'num_groups': num_groups, + 'block_class': BottleneckBlock + } + if stage_idx == 5: + stage_kargs.pop('dilation') + stage_kargs['dilation_per_block'] = [ + dilation * mg for mg in res5_multi_grid + ] + blocks = DeeplabResNet.make_stage(**stage_kargs) + in_channels = out_channels + out_channels *= 2 + bottleneck_channels *= 2 + stages.append(blocks) + return DeeplabResNet(stem, stages, out_features=out_features) diff --git a/modelscope/models/cv/image_instance_segmentation/fastinst/__init__.py b/modelscope/models/cv/image_instance_segmentation/fastinst/__init__.py new file mode 100644 index 00000000..b937315b --- /dev/null +++ b/modelscope/models/cv/image_instance_segmentation/fastinst/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. diff --git a/modelscope/models/cv/image_instance_segmentation/fastinst/fastinst_decoder.py b/modelscope/models/cv/image_instance_segmentation/fastinst/fastinst_decoder.py new file mode 100644 index 00000000..aa4300f6 --- /dev/null +++ b/modelscope/models/cv/image_instance_segmentation/fastinst/fastinst_decoder.py @@ -0,0 +1,351 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import math + +import torch +from torch import nn +from torch.nn import functional as F + +from modelscope.models.cv.image_colorization.ddcolor.utils.transformer_utils import ( + MLP, CrossAttentionLayer, FFNLayer, SelfAttentionLayer) + + +class QueryProposal(nn.Module): + + def __init__(self, num_features, num_queries, num_classes): + super().__init__() + self.topk = num_queries + self.num_classes = num_classes + + self.conv_proposal_cls_logits = nn.Sequential( + nn.Conv2d( + num_features, num_features, kernel_size=3, stride=1, + padding=1), + nn.ReLU(inplace=True), + nn.Conv2d( + num_features, + num_classes + 1, + kernel_size=1, + stride=1, + padding=0), + ) + + @torch.no_grad() + def compute_coordinates(self, x): + h, w = x.size(2), x.size(3) + y_loc = torch.linspace(0, 1, h, device=x.device) + x_loc = torch.linspace(0, 1, w, device=x.device) + y_loc, x_loc = torch.meshgrid(y_loc, x_loc) + locations = torch.stack([x_loc, y_loc], 0).unsqueeze(0) + return locations + + def seek_local_maximum(self, x, epsilon=1e-6): + """ + inputs: + x: torch.tensor, shape [b, c, h, w] + return: + torch.tensor, shape [b, c, h, w] + """ + x_pad = F.pad(x, (1, 1, 1, 1), 'constant', 0) + # top, bottom, left, right, top-left, top-right, bottom-left, bottom-right + maximum = (x >= x_pad[:, :, :-2, 1:-1]) & \ + (x >= x_pad[:, :, 2:, 1:-1]) & \ + (x >= x_pad[:, :, 1:-1, :-2]) & \ + (x >= x_pad[:, :, 1:-1, 2:]) & \ + (x >= x_pad[:, :, :-2, :-2]) & \ + (x >= x_pad[:, :, :-2, 2:]) & \ + (x >= x_pad[:, :, 2:, :-2]) & \ + (x >= x_pad[:, :, 2:, 2:]) & \ + (x >= epsilon) + return maximum.to(x) + + def forward(self, x, pos_embeddings): + + proposal_cls_logits = self.conv_proposal_cls_logits(x) # b, c, h, w + proposal_cls_probs = proposal_cls_logits.softmax(dim=1) # b, c, h, w + proposal_cls_one_hot = F.one_hot( + proposal_cls_probs[:, :-1, :, :].max(1)[1], + num_classes=self.num_classes + 1).permute(0, 3, 1, 2) # b, c, h, w + proposal_cls_probs = proposal_cls_probs.mul(proposal_cls_one_hot) + proposal_local_maximum_map = self.seek_local_maximum( + proposal_cls_probs) # b, c, h, w + proposal_cls_probs = proposal_cls_probs + proposal_local_maximum_map # b, c, h, w + + # top-k indices + topk_indices = torch.topk( + proposal_cls_probs[:, :-1, :, :].flatten(2).max(1)[0], + self.topk, + dim=1)[1] # b, q + topk_indices = topk_indices.unsqueeze(1) # b, 1, q + + # topk queries + topk_proposals = torch.gather( + x.flatten(2), dim=2, index=topk_indices.repeat(1, x.shape[1], + 1)) # b, c, q + pos_embeddings = pos_embeddings.repeat(x.shape[0], 1, 1, 1).flatten(2) + topk_pos_embeddings = torch.gather( + pos_embeddings, + dim=2, + index=topk_indices.repeat(1, pos_embeddings.shape[1], + 1)) # b, c, q + if self.training: + locations = self.compute_coordinates(x).repeat(x.shape[0], 1, 1, 1) + topk_locations = torch.gather( + locations.flatten(2), + dim=2, + index=topk_indices.repeat(1, locations.shape[1], 1)) + topk_locations = topk_locations.transpose(-1, -2) # b, q, 2 + else: + topk_locations = None + return topk_proposals, topk_pos_embeddings, topk_locations, proposal_cls_logits + + +class FastInstDecoder(nn.Module): + + def __init__(self, in_channels, *, num_classes: int, hidden_dim: int, + num_queries: int, num_aux_queries: int, nheads: int, + dim_feedforward: int, dec_layers: int, pre_norm: bool, + mask_dim: int): + """ + Args: + in_channels: channels of the input features + num_classes: number of classes + hidden_dim: Transformer feature dimension + num_queries: number of queries + num_aux_queries: number of auxiliary queries + nheads: number of heads + dim_feedforward: feature dimension in feedforward network + dec_layers: number of Transformer decoder layers + pre_norm: whether to use pre-LayerNorm or not + mask_dim: mask feature dimension + """ + super().__init__() + self.num_heads = nheads + self.num_layers = dec_layers + self.num_queries = num_queries + self.num_aux_queries = num_aux_queries + self.num_classes = num_classes + + meta_pos_size = int(round(math.sqrt(self.num_queries))) + self.meta_pos_embed = nn.Parameter( + torch.empty(1, hidden_dim, meta_pos_size, meta_pos_size)) + if num_aux_queries > 0: + self.empty_query_features = nn.Embedding(num_aux_queries, + hidden_dim) + self.empty_query_pos_embed = nn.Embedding(num_aux_queries, + hidden_dim) + + self.query_proposal = QueryProposal(hidden_dim, num_queries, + num_classes) + + self.transformer_query_cross_attention_layers = nn.ModuleList() + self.transformer_query_self_attention_layers = nn.ModuleList() + self.transformer_query_ffn_layers = nn.ModuleList() + self.transformer_mask_cross_attention_layers = nn.ModuleList() + self.transformer_mask_ffn_layers = nn.ModuleList() + for idx in range(self.num_layers): + self.transformer_query_cross_attention_layers.append( + CrossAttentionLayer( + d_model=hidden_dim, + nhead=nheads, + dropout=0.0, + normalize_before=pre_norm)) + self.transformer_query_self_attention_layers.append( + SelfAttentionLayer( + d_model=hidden_dim, + nhead=nheads, + dropout=0.0, + normalize_before=pre_norm)) + self.transformer_query_ffn_layers.append( + FFNLayer( + d_model=hidden_dim, + dim_feedforward=dim_feedforward, + dropout=0.0, + normalize_before=pre_norm)) + self.transformer_mask_cross_attention_layers.append( + CrossAttentionLayer( + d_model=hidden_dim, + nhead=nheads, + dropout=0.0, + normalize_before=pre_norm)) + self.transformer_mask_ffn_layers.append( + FFNLayer( + d_model=hidden_dim, + dim_feedforward=dim_feedforward, + dropout=0.0, + normalize_before=pre_norm)) + + self.decoder_query_norm_layers = nn.ModuleList() + self.class_embed_layers = nn.ModuleList() + self.mask_embed_layers = nn.ModuleList() + self.mask_features_layers = nn.ModuleList() + for idx in range(self.num_layers + 1): + self.decoder_query_norm_layers.append(nn.LayerNorm(hidden_dim)) + self.class_embed_layers.append( + MLP(hidden_dim, hidden_dim, num_classes + 1, 3)) + self.mask_embed_layers.append( + MLP(hidden_dim, hidden_dim, mask_dim, 3)) + self.mask_features_layers.append(nn.Linear(hidden_dim, mask_dim)) + + def forward(self, x, mask_features, targets=None): + bs = x[0].shape[0] + proposal_size = x[1].shape[-2:] + pixel_feature_size = x[2].shape[-2:] + + pixel_pos_embeds = F.interpolate( + self.meta_pos_embed, + size=pixel_feature_size, + mode='bilinear', + align_corners=False) + proposal_pos_embeds = F.interpolate( + self.meta_pos_embed, + size=proposal_size, + mode='bilinear', + align_corners=False) + + pixel_features = x[2].flatten(2).permute(2, 0, 1) + pixel_pos_embeds = pixel_pos_embeds.flatten(2).permute(2, 0, 1) + + query_features, query_pos_embeds, query_locations, proposal_cls_logits = self.query_proposal( + x[1], proposal_pos_embeds) + query_features = query_features.permute(2, 0, 1) + query_pos_embeds = query_pos_embeds.permute(2, 0, 1) + if self.num_aux_queries > 0: + aux_query_features = self.empty_query_features.weight.unsqueeze( + 1).repeat(1, bs, 1) + aux_query_pos_embed = self.empty_query_pos_embed.weight.unsqueeze( + 1).repeat(1, bs, 1) + query_features = torch.cat([query_features, aux_query_features], + dim=0) + query_pos_embeds = torch.cat( + [query_pos_embeds, aux_query_pos_embed], dim=0) + + outputs_class, outputs_mask, attn_mask, _, _ = self.forward_prediction_heads( + query_features, + pixel_features, + pixel_feature_size, + -1, + return_attn_mask=True) + predictions_class = [outputs_class] + predictions_mask = [outputs_mask] + predictions_matching_index = [None] + query_feature_memory = [query_features] + pixel_feature_memory = [pixel_features] + + for i in range(self.num_layers): + query_features, pixel_features = self.forward_one_layer( + query_features, pixel_features, query_pos_embeds, + pixel_pos_embeds, attn_mask, i) + if i < self.num_layers - 1: + outputs_class, outputs_mask, attn_mask, _, _ = self.forward_prediction_heads( + query_features, + pixel_features, + pixel_feature_size, + i, + return_attn_mask=True, + ) + else: + outputs_class, outputs_mask, _, matching_indices, gt_attn_mask = self.forward_prediction_heads( + query_features, + pixel_features, + pixel_feature_size, + i, + ) + predictions_class.append(outputs_class) + predictions_mask.append(outputs_mask) + predictions_matching_index.append(None) + query_feature_memory.append(query_features) + pixel_feature_memory.append(pixel_features) + + out = { + 'proposal_cls_logits': + proposal_cls_logits, + 'query_locations': + query_locations, + 'pred_logits': + predictions_class[-1], + 'pred_masks': + predictions_mask[-1], + 'pred_indices': + predictions_matching_index[-1], + 'aux_outputs': + self._set_aux_loss(predictions_class, predictions_mask, + predictions_matching_index, query_locations) + } + return out + + def forward_one_layer(self, query_features, pixel_features, + query_pos_embeds, pixel_pos_embeds, attn_mask, i): + pixel_features = self.transformer_mask_cross_attention_layers[i]( + pixel_features, + query_features, + query_pos=pixel_pos_embeds, + pos=query_pos_embeds) + pixel_features = self.transformer_mask_ffn_layers[i](pixel_features) + + query_features = self.transformer_query_cross_attention_layers[i]( + query_features, + pixel_features, + memory_mask=attn_mask, + query_pos=query_pos_embeds, + pos=pixel_pos_embeds) + query_features = self.transformer_query_self_attention_layers[i]( + query_features, query_pos=query_pos_embeds) + query_features = self.transformer_query_ffn_layers[i](query_features) + return query_features, pixel_features + + def forward_prediction_heads(self, + query_features, + pixel_features, + pixel_feature_size, + idx_layer, + return_attn_mask=False, + return_gt_attn_mask=False, + targets=None, + query_locations=None): + decoder_query_features = self.decoder_query_norm_layers[idx_layer + 1]( + query_features[:self.num_queries]) + decoder_query_features = decoder_query_features.transpose(0, 1) + if idx_layer + 1 == self.num_layers: + outputs_class = self.class_embed_layers[idx_layer + 1]( + decoder_query_features) + else: + outputs_class = None + outputs_mask_embed = self.mask_embed_layers[idx_layer + 1]( + decoder_query_features) + outputs_mask_features = self.mask_features_layers[idx_layer + 1]( + pixel_features.transpose(0, 1)) + + outputs_mask = torch.einsum('bqc,blc->bql', outputs_mask_embed, + outputs_mask_features) + outputs_mask = outputs_mask.reshape(-1, self.num_queries, + *pixel_feature_size) + + if return_attn_mask: + # outputs_mask.shape: b, q, h, w + attn_mask = F.pad(outputs_mask, + (0, 0, 0, 0, 0, self.num_aux_queries), + 'constant', 1) + attn_mask = (attn_mask < 0.).flatten(2) # b, q, hw + invalid_query = attn_mask.all(-1, keepdim=True) # b, q, 1 + attn_mask = (~invalid_query) & attn_mask # b, q, hw + attn_mask = attn_mask.unsqueeze(1).repeat(1, self.num_heads, 1, + 1).flatten(0, 1) + attn_mask = attn_mask.detach() + else: + attn_mask = None + + matching_indices = None + gt_attn_mask = None + + return outputs_class, outputs_mask, attn_mask, matching_indices, gt_attn_mask + + @torch.jit.unused + def _set_aux_loss(self, outputs_class, outputs_seg_masks, output_indices, + output_query_locations): + return [{ + 'query_locations': output_query_locations, + 'pred_logits': a, + 'pred_masks': b, + 'pred_matching_indices': c + } for a, b, c in zip(outputs_class[:-1], outputs_seg_masks[:-1], + output_indices[:-1])] diff --git a/modelscope/models/cv/image_instance_segmentation/fastinst/fastinst_encoder.py b/modelscope/models/cv/image_instance_segmentation/fastinst/fastinst_encoder.py new file mode 100644 index 00000000..46b3f74d --- /dev/null +++ b/modelscope/models/cv/image_instance_segmentation/fastinst/fastinst_encoder.py @@ -0,0 +1,180 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import logging +from typing import Callable, Optional, Union + +import torch +from torch import nn +from torch.nn import functional as F + +from modelscope.models.cv.image_instance_segmentation.maskdino.utils import \ + Conv2d + + +# This is a modified FPN decoder. +class BaseFPN(nn.Module): + + def __init__( + self, + input_shape, + *, + convs_dim: int, + mask_dim: int, + norm: Optional[Union[str, Callable]] = None, + ): + """ + Args: + input_shape: shapes (channels and stride) of the input features + convs_dim: number of output channels for the intermediate conv layers. + mask_dim: number of output channels for the final conv layer. + norm (str or callable): normalization for all conv layers + """ + super().__init__() + + input_shape = sorted(input_shape.items(), key=lambda x: x[1]['stride']) + self.in_features = [k for k, v in input_shape + ] # starting from "res3" to "res5" + feature_channels = [v['channels'] for k, v in input_shape] + + lateral_convs = [] + output_convs = [] + + use_bias = norm == '' + for idx, in_channels in enumerate(feature_channels): + lateral_norm = nn.GroupNorm(32, convs_dim) + output_norm = nn.GroupNorm(32, convs_dim) + + lateral_conv = Conv2d( + in_channels, + convs_dim, + kernel_size=1, + bias=use_bias, + norm=lateral_norm) + output_conv = Conv2d( + convs_dim, + convs_dim, + kernel_size=3, + stride=1, + padding=1, + bias=use_bias, + norm=output_norm, + activation=F.relu, + ) + self.add_module('adapter_{}'.format(idx + 1), lateral_conv) + self.add_module('layer_{}'.format(idx + 1), output_conv) + + lateral_convs.append(lateral_conv) + output_convs.append(output_conv) + # Place convs into top-down order (from low to high resolution) + # to make the top-down computation in forward clearer. + self.lateral_convs = lateral_convs[::-1] + self.output_convs = output_convs[::-1] + + self.convs_dim = convs_dim + self.num_feature_levels = 3 # always use 3 scales + + def forward_features(self, features): + multi_scale_features = [] + num_cur_levels = 0 + # Reverse feature maps into top-down order (from low to high resolution) + for idx, f in enumerate(self.in_features[::-1]): + x = features[f] + lateral_conv = self.lateral_convs[idx] + output_conv = self.output_convs[idx] + if idx == 0: + y = lateral_conv(x) + else: + cur_fpn = lateral_conv(x) + y = cur_fpn + F.interpolate( + y, + size=cur_fpn.shape[-2:], + mode='bilinear', + align_corners=False) + y = output_conv(y) + + if num_cur_levels < self.num_feature_levels: + multi_scale_features.append(y) + num_cur_levels += 1 + return None, multi_scale_features + + def forward(self, features, targets=None): + logger = logging.getLogger(__name__) + logger.warning( + 'Calling forward() may cause unpredicted behavior of PixelDecoder module.' + ) + return self.forward_features(features) + + +class PyramidPoolingModule(nn.Module): + + def __init__(self, in_channels, channels=512, sizes=(1, 2, 3, 6)): + super().__init__() + self.stages = [] + self.stages = nn.ModuleList( + [self._make_stage(in_channels, channels, size) for size in sizes]) + self.bottleneck = Conv2d(in_channels + len(sizes) * channels, + in_channels, 1) + + def _make_stage(self, features, out_features, size): + prior = nn.AdaptiveAvgPool2d(output_size=(size, size)) + conv = Conv2d(features, out_features, 1) + return nn.Sequential(prior, conv) + + def forward(self, feats): + h, w = feats.size(2), feats.size(3) + priors = [ + F.interpolate( + input=F.relu_(stage(feats)), + size=(h, w), + mode='bilinear', + align_corners=False) for stage in self.stages + ] + [feats] + out = F.relu_(self.bottleneck(torch.cat(priors, 1))) + return out + + +class PyramidPoolingModuleFPN(BaseFPN): + + def __init__( + self, + input_shape, + *, + convs_dim: int, + mask_dim: int, + norm: Optional[Union[str, Callable]] = None, + ): + """ + NOTE: this interface is experimental. + Args: + input_shape: shapes (channels and stride) of the input features + convs_dim: number of output channels for the intermediate conv layers. + mask_dim: number of output channels for the final conv layer. + norm (str or callable): normalization for all conv layers + """ + super().__init__( + input_shape, convs_dim=convs_dim, mask_dim=mask_dim, norm=norm) + self.ppm = PyramidPoolingModule(convs_dim, convs_dim // 4) + + def forward_features(self, features): + multi_scale_features = [] + num_cur_levels = 0 + # Reverse feature maps into top-down order (from low to high resolution) + for idx, f in enumerate(self.in_features[::-1]): + x = features[f] + lateral_conv = self.lateral_convs[idx] + output_conv = self.output_convs[idx] + if idx == 0: + y = self.ppm(lateral_conv(x)) + else: + cur_fpn = lateral_conv(x) + y = cur_fpn + F.interpolate( + y, + size=cur_fpn.shape[-2:], + mode='bilinear', + align_corners=False) + y = output_conv(y) + + if num_cur_levels < self.num_feature_levels: + multi_scale_features.append(y) + num_cur_levels += 1 + + return None, multi_scale_features diff --git a/modelscope/models/cv/image_instance_segmentation/fastinst_model.py b/modelscope/models/cv/image_instance_segmentation/fastinst_model.py new file mode 100644 index 00000000..f9cfbc4f --- /dev/null +++ b/modelscope/models/cv/image_instance_segmentation/fastinst_model.py @@ -0,0 +1,221 @@ +# Part of implementation is borrowed and modified from Mask2Former, publicly available at +# https://github.com/facebookresearch/Mask2Former. +import os +from typing import Any, Dict, List + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from modelscope.metainfo import Models +from modelscope.models.base import TorchModel +from modelscope.models.builder import MODELS +from modelscope.models.cv.image_instance_segmentation.maskdino_swin import \ + ImageList +from modelscope.utils.constant import ModelFile, Tasks +from modelscope.utils.logger import get_logger +from .backbones import build_resnet_backbone +from .fastinst.fastinst_decoder import FastInstDecoder +from .fastinst.fastinst_encoder import PyramidPoolingModuleFPN + +logger = get_logger() + + +@MODELS.register_module(Tasks.image_segmentation, module_name=Models.fastinst) +class FastInst(TorchModel): + + def __init__(self, + model_dir, + backbone=None, + encoder=None, + decoder=None, + pretrained=None, + classes=None, + **kwargs): + """ + Deep Learning Technique for Human Parsing: A Survey and Outlook. See https://arxiv.org/abs/2301.00394 + Args: + backbone (dict): backbone config. + encoder (dict): encoder config. + decoder (dict): decoder config. + pretrained (bool): whether to use pretrained model + classes (list): class names + """ + super(FastInst, self).__init__(model_dir, **kwargs) + + self.backbone = build_resnet_backbone( + **backbone, input_shape={'channels': 3}) + in_features = encoder.pop('in_features') + input_shape = { + k: v + for k, v in self.backbone.output_shape().items() + if k in in_features + } + encoder = PyramidPoolingModuleFPN(input_shape=input_shape, **encoder) + decoder = FastInstDecoder(in_channels=encoder.convs_dim, **decoder) + self.sem_seg_head = FastInstHead( + pixel_decoder=encoder, transformer_predictor=decoder) + + self.num_classes = decoder.num_classes + self.num_queries = decoder.num_queries + self.size_divisibility = 32 + self.register_buffer( + 'pixel_mean', + torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1), False) + self.register_buffer( + 'pixel_std', + torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1), False) + self.classes = classes + self.test_topk_per_image = 100 + + if pretrained: + model_path = os.path.join(model_dir, ModelFile.TORCH_MODEL_FILE) + logger.info(f'loading model from {model_path}') + weight = torch.load(model_path, map_location='cpu')['model'] + tgt_weight = self.state_dict() + for name in list(weight.keys()): + if name in tgt_weight: + load_size = weight[name].size() + tgt_size = tgt_weight[name].size() + mis_match = False + if len(load_size) != len(tgt_size): + mis_match = True + else: + for n1, n2 in zip(load_size, tgt_size): + if n1 != n2: + mis_match = True + break + if mis_match: + logger.info( + f'size mismatch for {name} ' + f'({load_size} -> {tgt_size}), skip loading.') + del weight[name] + else: + logger.info( + f'{name} doesn\'t exist in current model, skip loading.' + ) + + self.load_state_dict(weight, strict=False) + logger.info('load model done') + + def forward(self, batched_inputs: List[dict]) -> Dict[str, Any]: + images = [x['image'].to(self.device) for x in batched_inputs] + images = [(x - self.pixel_mean) / self.pixel_std for x in images] + images = ImageList.from_tensors(images, self.size_divisibility) + + features = self.backbone(images.tensor) + outputs = self.sem_seg_head(features) + + return dict( + outputs=outputs, batched_inputs=batched_inputs, images=images) + + def postprocess(self, input: Dict[str, Any]) -> Dict[str, Any]: + outputs = input['outputs'] + batched_inputs = input['batched_inputs'] + images = input['images'] + if self.training: + raise NotImplementedError + else: + mask_cls_results = outputs['pred_logits'] # (B, Q, C+1) + mask_pred_results = outputs['pred_masks'] # (B, Q, H, W) + # upsample masks + mask_pred_results = F.interpolate( + mask_pred_results, + size=(images.tensor.shape[-2], images.tensor.shape[-1]), + mode='bilinear', + align_corners=False, + ) + + del outputs + + processed_results = [] + for mask_cls_result, mask_pred_result, input_per_image, image_size in zip( + mask_cls_results, mask_pred_results, batched_inputs, + images.image_sizes): + height = input_per_image.get('height', image_size[0]) + width = input_per_image.get('width', image_size[1]) + processed_results.append({}) # for each image + + mask_pred_result = self.sem_seg_postprocess( + mask_pred_result, image_size, height, width) + mask_cls_result = mask_cls_result.to(mask_pred_result) + + instance_r = self.instance_inference(mask_cls_result, + mask_pred_result) + processed_results[-1]['instances'] = instance_r + + return dict(eval_result=processed_results) + + @property + def device(self): + return self.pixel_mean.device + + def sem_seg_postprocess(self, result, img_size, output_height, + output_width): + result = result[:, :img_size[0], :img_size[1]].expand(1, -1, -1, -1) + result = F.interpolate( + result, + size=(output_height, output_width), + mode='bilinear', + align_corners=False)[0] + return result + + def instance_inference(self, mask_cls, mask_pred): + # mask_pred is already processed to have the same shape as original input + image_size = mask_pred.shape[-2:] + + # [Q, K] + scores = F.softmax(mask_cls, dim=-1)[:, :-1] + labels = torch.arange( + self.num_classes, + device=self.device).unsqueeze(0).repeat(self.num_queries, + 1).flatten(0, 1) + scores_per_image, topk_indices = scores.flatten(0, 1).topk( + self.test_topk_per_image, sorted=False) + labels_per_image = labels[topk_indices] + + topk_indices = topk_indices // self.num_classes + mask_pred = mask_pred[topk_indices] + + result = {'image_size': image_size} + # mask (before sigmoid) + mask_pred_sigmoid = mask_pred.sigmoid() + result['pred_masks'] = (mask_pred_sigmoid > 0.5).float() + + # calculate average mask prob + mask_scores_per_image = (mask_pred_sigmoid.flatten(1) + * result['pred_masks'].flatten(1)).sum(1) / ( + result['pred_masks'].flatten(1).sum(1) + + 1e-6) + result['scores'] = scores_per_image * mask_scores_per_image + result['pred_classes'] = labels_per_image + return result + + +class FastInstHead(nn.Module): + + def __init__( + self, + *, + pixel_decoder: nn.Module, + # extra parameters + transformer_predictor: nn.Module): + """ + NOTE: this interface is experimental. + Args: + pixel_decoder: the pixel decoder module + transformer_predictor: the transformer decoder that makes prediction + """ + super().__init__() + self.pixel_decoder = pixel_decoder + self.predictor = transformer_predictor + + def forward(self, features, targets=None): + return self.layers(features, targets) + + def layers(self, features, targets=None): + mask_features, multi_scale_features = self.pixel_decoder.forward_features( + features) + predictions = self.predictor(multi_scale_features, mask_features, + targets) + return predictions diff --git a/modelscope/pipelines/audio/speaker_change_locating_pipeline.py b/modelscope/pipelines/audio/speaker_change_locating_pipeline.py new file mode 100644 index 00000000..0bab08ac --- /dev/null +++ b/modelscope/pipelines/audio/speaker_change_locating_pipeline.py @@ -0,0 +1,105 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import io +from typing import Any, Dict, List, Union + +import numpy as np +import soundfile as sf +import torch + +from modelscope.fileio import File +from modelscope.metainfo import Pipelines +from modelscope.outputs import OutputKeys +from modelscope.pipelines.base import InputModel, Pipeline +from modelscope.pipelines.builder import PIPELINES +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['SpeakerChangeLocatingPipeline'] + + +@PIPELINES.register_module( + Tasks.speaker_diarization, module_name=Pipelines.speaker_change_locating) +class SpeakerChangeLocatingPipeline(Pipeline): + """Speaker Change Locating Inference Pipeline + use `model` to create a speaker change Locating pipeline. + + Args: + model (SpeakerChangeLocatingPipeline): A model instance, or a model local dir, or a model id in the model hub. + kwargs (dict, `optional`): + Extra kwargs passed into the pipeline's constructor. + Example: + >>> from modelscope.pipelines import pipeline + >>> from modelscope.utils.constant import Tasks + >>> p = pipeline( + >>> task=Tasks.speaker_diarization, model='damo/speech_campplus-transformer_scl_zh-cn_16k-common') + >>> print(p(audio)) + + """ + + def __init__(self, model: InputModel, **kwargs): + """use `model` to create a speaker change Locating pipeline for prediction + Args: + model (str): a valid offical model id + """ + super().__init__(model=model, **kwargs) + self.model_config = self.model.model_config + self.config = self.model.model_config + self.anchor_size = self.config['anchor_size'] + + def __call__(self, audio: str, embds: List = None) -> Dict[str, Any]: + if embds is not None: + assert len(embds) == 2 + assert isinstance(embds[0], np.ndarray) and isinstance( + embds[1], np.ndarray) + assert embds[0].shape == ( + self.anchor_size, ) and embds[1].shape == (self.anchor_size, ) + else: + embd1 = np.zeros(self.anchor_size // 2) + embd2 = np.ones(self.anchor_size - self.anchor_size // 2) + embd3 = np.ones(self.anchor_size // 2) + embd4 = np.zeros(self.anchor_size - self.anchor_size // 2) + embds = [ + np.stack([embd1, embd2], axis=1).flatten(), + np.stack([embd3, embd4], axis=1).flatten(), + ] + anchors = torch.from_numpy(np.stack(embds, + axis=0)).float().unsqueeze(0) + + output = self.preprocess(audio) + output = self.forward(output, anchors) + output = self.postprocess(output) + + return output + + def forward(self, input: torch.Tensor, anchors: torch.Tensor): + output = self.model(input, anchors) + return output + + def postprocess(self, input: torch.Tensor) -> Dict[str, Any]: + predict = np.where(np.diff(input.argmax(-1).numpy())) + try: + predict = predict[0][0] * 0.01 + 0.02 + predict = round(predict, 2) + return {OutputKeys.TEXT: f'The change point is at {predict}s.'} + except Exception: + return {OutputKeys.TEXT: 'No change point is found.'} + + def preprocess(self, input: str) -> torch.Tensor: + if isinstance(input, str): + file_bytes = File.read(input) + data, fs = sf.read(io.BytesIO(file_bytes), dtype='float32') + if len(data.shape) == 2: + data = data[:, 0] + if fs != self.model_config['sample_rate']: + raise ValueError( + 'modelscope error: Only support %d sample rate files' + % self.model_cfg['sample_rate']) + data = torch.from_numpy(data).unsqueeze(0) + else: + raise ValueError( + 'modelscope error: The input type is restricted to audio file address' + % i) + return data diff --git a/modelscope/pipelines/cv/fast_instance_segmentation_pipeline.py b/modelscope/pipelines/cv/fast_instance_segmentation_pipeline.py new file mode 100644 index 00000000..6ee341de --- /dev/null +++ b/modelscope/pipelines/cv/fast_instance_segmentation_pipeline.py @@ -0,0 +1,116 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Any, Dict, Optional, Union + +import numpy as np +import torch +import torchvision.transforms as T + +from modelscope.metainfo import Pipelines +from modelscope.models.cv.image_instance_segmentation import FastInst +from modelscope.outputs import OutputKeys +from modelscope.pipelines.base import Input, Pipeline +from modelscope.pipelines.builder import PIPELINES +from modelscope.preprocessors import LoadImage +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +@PIPELINES.register_module( + Tasks.image_segmentation, module_name=Pipelines.fast_instance_segmentation) +class FastInstanceSegmentationPipeline(Pipeline): + + def __init__(self, + model: Union[FastInst, str], + preprocessor: Optional = None, + **kwargs): + r"""The inference pipeline for fastinst models. + + The model outputs a dict with keys of `scores`, `labels`, and `masks`. + + Args: + model (`str` or `Model` or module instance): A model instance or a model local dir + or a model id in the model hub. + preprocessor (`Preprocessor`, `optional`): A Preprocessor instance. + kwargs (dict, `optional`): + Extra kwargs passed into the preprocessor's constructor. + + Examples: + >>> from modelscope.outputs import OutputKeys + >>> from modelscope.pipelines import pipeline + >>> pipeline_ins = pipeline('image-segmentation', + model='damo/cv_resnet50_fast-instance-segmentation_coco') + >>> input_img = 'https://modelscope.oss-cn-beijing.aliyuncs.com/test/images/image_instance_segmentation.jpg' + >>> print(pipeline_ins(input_img)[OutputKeys.LABELS]) + """ + super().__init__(model=model, preprocessor=preprocessor, **kwargs) + self.model.eval() + + def _get_preprocess_shape(self, oldh, oldw, short_edge_length, max_size): + h, w = oldh, oldw + size = short_edge_length * 1.0 + scale = size / min(h, w) + if h < w: + newh, neww = size, scale * w + else: + newh, neww = scale * h, size + if max(newh, neww) > max_size: + scale = max_size * 1.0 / max(newh, neww) + newh = newh * scale + neww = neww * scale + neww = int(neww + 0.5) + newh = int(newh + 0.5) + return (newh, neww) + + def preprocess(self, + input: Input, + min_size=640, + max_size=1333) -> Dict[str, Any]: + image = LoadImage.convert_to_img(input) + w, h = image.size[:2] + dataset_dict = {'width': w, 'height': h} + new_h, new_w = self._get_preprocess_shape(h, w, min_size, max_size) + test_transforms = T.Compose([ + T.Resize((new_h, new_w)), + T.ToTensor(), + ]) + image = test_transforms(image) + dataset_dict['image'] = image * 255. + result = {'batched_inputs': [dataset_dict]} + return result + + def forward(self, input: Dict[str, Any], + **forward_params) -> Dict[str, Any]: + with torch.no_grad(): + output = self.model(**input) + return output + + def postprocess(self, + inputs: Dict[str, Any], + score_thr=0.5) -> Dict[str, Any]: + predictions = inputs['eval_result'][0]['instances'] + scores = predictions['scores'].detach().cpu().numpy() + pred_masks = predictions['pred_masks'].detach().cpu().numpy() + pred_classes = predictions['pred_classes'].detach().cpu().numpy() + + thresholded_idxs = np.array(scores) >= score_thr + scores = scores[thresholded_idxs] + pred_classes = pred_classes[thresholded_idxs] + pred_masks = pred_masks[thresholded_idxs] + + results_dict = { + OutputKeys.MASKS: [], + OutputKeys.LABELS: [], + OutputKeys.SCORES: [] + } + for score, cls, mask in zip(scores, pred_classes, pred_masks): + score = np.float64(score) + label = self.model.classes[int(cls)] + mask = np.array(mask, dtype=np.float64) + + results_dict[OutputKeys.SCORES].append(score) + results_dict[OutputKeys.LABELS].append(label) + results_dict[OutputKeys.MASKS].append(mask) + + return results_dict diff --git a/tests/pipelines/test_fast_instance_segmentation.py b/tests/pipelines/test_fast_instance_segmentation.py new file mode 100644 index 00000000..aefd1092 --- /dev/null +++ b/tests/pipelines/test_fast_instance_segmentation.py @@ -0,0 +1,39 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import unittest + +from modelscope.models import Model +from modelscope.outputs import OutputKeys +from modelscope.pipelines import pipeline +from modelscope.utils.constant import Tasks +from modelscope.utils.demo_utils import DemoCompatibilityCheck +from modelscope.utils.test_utils import test_level + + +class FastInstanceSegmentationTest(unittest.TestCase, DemoCompatibilityCheck): + + def setUp(self) -> None: + self.task = Tasks.image_segmentation + self.model_id = 'damo/cv_resnet50_fast-instance-segmentation_coco' + + image = 'data/test/images/image_instance_segmentation.jpg' + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_model_name(self): + pipeline_parsing = pipeline( + task=Tasks.image_segmentation, model=self.model_id) + print(pipeline_parsing(input=self.image)[OutputKeys.LABELS]) + + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') + def test_run_with_model_from_modelhub(self): + model = Model.from_pretrained(self.model_id) + pipeline_parsing = pipeline( + task=Tasks.image_segmentation, model=model, preprocessor=None) + print(pipeline_parsing(input=self.image)[OutputKeys.LABELS]) + + @unittest.skip('demo compatibility test is only enabled on a needed-basis') + def test_demo_compatibility(self): + self.compatibility_check() + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/pipelines/test_speaker_verification.py b/tests/pipelines/test_speaker_verification.py index 236b952b..41335fc8 100644 --- a/tests/pipelines/test_speaker_verification.py +++ b/tests/pipelines/test_speaker_verification.py @@ -2,7 +2,7 @@ import os.path import unittest -from typing import Any, Dict, List +from typing import Any, Dict, List, Union from modelscope.outputs import OutputKeys from modelscope.pipelines import pipeline @@ -16,20 +16,25 @@ logger = get_logger() SPEAKER1_A_EN_16K_WAV = 'data/test/audios/speaker1_a_en_16k.wav' SPEAKER1_B_EN_16K_WAV = 'data/test/audios/speaker1_b_en_16k.wav' SPEAKER2_A_EN_16K_WAV = 'data/test/audios/speaker2_a_en_16k.wav' +SCL_EXAMPLE_WAV = 'data/test/audios/scl_example1.wav' class SpeakerVerificationTest(unittest.TestCase, DemoCompatibilityCheck): ecapatdnn_voxceleb_16k_model_id = 'damo/speech_ecapa-tdnn_sv_en_voxceleb_16k' campplus_voxceleb_16k_model_id = 'damo/speech_campplus_sv_en_voxceleb_16k' rdino_voxceleb_16k_model_id = 'damo/speech_rdino_ecapa_tdnn_sv_en_voxceleb_16k' + speaker_change_locating_cn_model_id = 'damo/speech_campplus-transformer_scl_zh-cn_16k-common' def setUp(self) -> None: self.task = Tasks.speaker_verification def run_pipeline(self, model_id: str, - audios: List[str], + audios: Union[List[str], str], + task: str = None, model_revision=None) -> Dict[str, Any]: + if task is not None: + self.task = task p = pipeline( task=self.task, model=model_id, model_revision=model_revision) result = p(audios) @@ -66,6 +71,17 @@ class SpeakerVerificationTest(unittest.TestCase, DemoCompatibilityCheck): print(result) self.assertTrue(OutputKeys.SCORE in result) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_speaker_change_locating_cn_16k(self): + logger.info( + 'Run speaker change locating for campplus-transformer model') + result = self.run_pipeline( + model_id=self.speaker_change_locating_cn_model_id, + task=Tasks.speaker_diarization, + audios=SCL_EXAMPLE_WAV) + print(result) + self.assertTrue(OutputKeys.TEXT in result) + @unittest.skip('demo compatibility test is only enabled on a needed-basis') def test_demo_compatibility(self): self.compatibility_check()