From 2ecb97ac4f0d7fde33a65f66ed2d59865da68cf7 Mon Sep 17 00:00:00 2001 From: yfchenmodelscope <160825272+yfchenmodelscope@users.noreply.github.com> Date: Fri, 23 Feb 2024 16:19:13 +0800 Subject: [PATCH] add res2net resnet models (#772) * add res2net resnet models * add paper link and model introduction --- modelscope/metainfo.py | 4 + modelscope/models/audio/sv/ERes2Net.py | 12 +- modelscope/models/audio/sv/Res2Net.py | 234 ++++++++++++++++++ modelscope/models/audio/sv/ResNet.py | 186 ++++++++++++++ .../speaker_verification_res2net_pipeline.py | 159 ++++++++++++ .../speaker_verification_resnet_pipeline.py | 159 ++++++++++++ tests/pipelines/test_speaker_verification.py | 27 +- 7 files changed, 772 insertions(+), 9 deletions(-) create mode 100644 modelscope/models/audio/sv/Res2Net.py create mode 100644 modelscope/models/audio/sv/ResNet.py create mode 100644 modelscope/pipelines/audio/speaker_verification_res2net_pipeline.py create mode 100644 modelscope/pipelines/audio/speaker_verification_resnet_pipeline.py diff --git a/modelscope/metainfo.py b/modelscope/metainfo.py index 837b3870..32a87f5d 100644 --- a/modelscope/metainfo.py +++ b/modelscope/metainfo.py @@ -204,6 +204,8 @@ class Models(object): ecapa_tdnn_sv = 'ecapa-tdnn-sv' campplus_sv = 'cam++-sv' eres2net_sv = 'eres2net-sv' + resnet_sv = 'resnet-sv' + res2net_sv = 'res2net-sv' eres2net_aug_sv = 'eres2net-aug-sv' scl_sd = 'scl-sd' scl_sd_xvector = 'scl-sd-xvector' @@ -550,6 +552,8 @@ class Pipelines(object): speaker_verification = 'speaker-verification' speaker_verification_rdino = 'speaker-verification-rdino' speaker_verification_eres2net = 'speaker-verification-eres2net' + speaker_verification_resnet = 'speaker-verification-resnet' + speaker_verification_res2net = 'speaker-verification-res2net' speech_language_recognition = 'speech-language-recognition' speech_language_recognition_eres2net = 'speech-language-recognition-eres2net' speaker_change_locating = 'speaker-change-locating' diff --git a/modelscope/models/audio/sv/ERes2Net.py b/modelscope/models/audio/sv/ERes2Net.py index 3c07390b..0d4a8137 100644 --- a/modelscope/models/audio/sv/ERes2Net.py +++ b/modelscope/models/audio/sv/ERes2Net.py @@ -55,11 +55,11 @@ def conv3x3(in_planes, out_planes, stride=1): bias=False) -class BasicBlockRes2Net(nn.Module): +class BasicBlockERes2Net(nn.Module): expansion = 2 def __init__(self, in_planes, planes, stride=1, baseWidth=32, scale=2): - super(BasicBlockRes2Net, self).__init__() + super(BasicBlockERes2Net, self).__init__() width = int(math.floor(planes * (baseWidth / 64.0))) self.conv1 = conv1x1(in_planes, width * scale, stride) self.bn1 = nn.BatchNorm2d(width * scale) @@ -118,11 +118,11 @@ class BasicBlockRes2Net(nn.Module): return out -class BasicBlockRes2Net_diff_AFF(nn.Module): +class BasicBlockERes2Net_AFF(nn.Module): expansion = 2 def __init__(self, in_planes, planes, stride=1, baseWidth=32, scale=2): - super(BasicBlockRes2Net_diff_AFF, self).__init__() + super(BasicBlockERes2Net_AFF, self).__init__() width = int(math.floor(planes * (baseWidth / 64.0))) self.conv1 = conv1x1(in_planes, width * scale, stride) self.bn1 = nn.BatchNorm2d(width * scale) @@ -190,8 +190,8 @@ class BasicBlockRes2Net_diff_AFF(nn.Module): class ERes2Net(nn.Module): def __init__(self, - block=BasicBlockRes2Net, - block_fuse=BasicBlockRes2Net_diff_AFF, + block=BasicBlockERes2Net, + block_fuse=BasicBlockERes2Net_AFF, num_blocks=[3, 4, 6, 3], m_channels=32, feat_dim=80, diff --git a/modelscope/models/audio/sv/Res2Net.py b/modelscope/models/audio/sv/Res2Net.py new file mode 100644 index 00000000..0d26e601 --- /dev/null +++ b/modelscope/models/audio/sv/Res2Net.py @@ -0,0 +1,234 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +""" Res2Net implementation is adapted from https://github.com/Res2Net/Res2Net-PretrainedModels. + Res2Net is an advanced neural network architecture that enhances the capabilities of standard ResNets + by incorporating hierarchical residual-like connections. This innovative structure improves + performance across various computer vision tasks, such as image classification and object + detection, without significant computational overhead. + Reference: https://arxiv.org/pdf/1904.01169.pdf + Some modifications from the original architecture: + 1. Smaller kernel size for the input layer + 2. Smaller expansion in BasicBlockRes2Net +""" +import math +import os +from typing import Any, Dict, Union + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchaudio.compliance.kaldi as Kaldi + +import modelscope.models.audio.sv.pooling_layers as pooling_layers +from modelscope.metainfo import Models +from modelscope.models import MODELS, TorchModel +from modelscope.utils.constant import Tasks +from modelscope.utils.device import create_device + + +class ReLU(nn.Hardtanh): + + def __init__(self, inplace=False): + super(ReLU, self).__init__(0, 20, inplace) + + def __repr__(self): + inplace_str = 'inplace' if self.inplace else '' + return self.__class__.__name__ + ' (' \ + + inplace_str + ')' + + +class BasicBlockRes2Net(nn.Module): + expansion = 2 + + def __init__(self, in_planes, planes, stride=1, baseWidth=32, scale=2): + super(BasicBlockRes2Net, self).__init__() + width = int(math.floor(planes * (baseWidth / 64.0))) + self.conv1 = nn.Conv2d( + in_planes, width * scale, kernel_size=1, stride=stride, bias=False) + self.bn1 = nn.BatchNorm2d(width * scale) + self.nums = scale - 1 + convs = [] + bns = [] + for i in range(self.nums): + convs.append( + nn.Conv2d(width, width, kernel_size=3, padding=1, bias=False)) + bns.append(nn.BatchNorm2d(width)) + self.convs = nn.ModuleList(convs) + self.bns = nn.ModuleList(bns) + self.relu = ReLU(inplace=True) + + self.conv3 = nn.Conv2d( + width * scale, planes * self.expansion, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion * planes: + self.shortcut = nn.Sequential( + nn.Conv2d( + in_planes, + self.expansion * planes, + kernel_size=1, + stride=stride, + bias=False), nn.BatchNorm2d(self.expansion * planes)) + self.stride = stride + self.width = width + self.scale = scale + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + spx = torch.split(out, self.width, 1) + for i in range(self.nums): + if i == 0: + sp = spx[i] + else: + sp = sp + spx[i] + sp = self.convs[i](sp) + sp = self.relu(self.bns[i](sp)) + if i == 0: + out = sp + else: + out = torch.cat((out, sp), 1) + + out = torch.cat((out, spx[self.nums]), 1) + + out = self.conv3(out) + out = self.bn3(out) + + residual = self.shortcut(x) + out += residual + out = self.relu(out) + + return out + + +class Res2Net(nn.Module): + + def __init__(self, + block=BasicBlockRes2Net, + num_blocks=[3, 4, 6, 3], + m_channels=32, + feat_dim=80, + embedding_size=192, + pooling_func='TSTP', + two_emb_layer=False): + super(Res2Net, self).__init__() + self.in_planes = m_channels + self.feat_dim = feat_dim + self.embedding_size = embedding_size + self.stats_dim = int(feat_dim / 8) * m_channels * 8 + self.two_emb_layer = two_emb_layer + + self.conv1 = nn.Conv2d( + 1, m_channels, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(m_channels) + + self.layer1 = self._make_layer( + block, m_channels, num_blocks[0], stride=1) + self.layer2 = self._make_layer( + block, m_channels * 2, num_blocks[1], stride=2) + self.layer3 = self._make_layer( + block, m_channels * 4, num_blocks[2], stride=2) + self.layer4 = self._make_layer( + block, m_channels * 8, num_blocks[3], stride=2) + + self.n_stats = 1 if pooling_func == 'TAP' or pooling_func == 'TSDP' else 2 + self.pool = getattr(pooling_layers, pooling_func)( + in_dim=self.stats_dim * block.expansion) + self.seg_1 = nn.Linear(self.stats_dim * block.expansion * self.n_stats, + embedding_size) + if self.two_emb_layer: + self.seg_bn_1 = nn.BatchNorm1d(embedding_size, affine=False) + self.seg_2 = nn.Linear(embedding_size, embedding_size) + else: + self.seg_bn_1 = nn.Identity() + self.seg_2 = nn.Identity() + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1] * (num_blocks - 1) + layers = [] + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + return nn.Sequential(*layers) + + def forward(self, x): + x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T) + + x = x.unsqueeze_(1) + out = F.relu(self.bn1(self.conv1(x))) + out = self.layer1(out) + out = self.layer2(out) + out = self.layer3(out) + out = self.layer4(out) + + stats = self.pool(out) + + embed_a = self.seg_1(stats) + if self.two_emb_layer: + out = F.relu(embed_a) + out = self.seg_bn_1(out) + embed_b = self.seg_2(out) + return embed_b + else: + return embed_a + + +@MODELS.register_module( + Tasks.speaker_verification, module_name=Models.res2net_sv) +class SpeakerVerificationResNet(TorchModel): + r""" + Args: + model_dir: A model dir. + model_config: The model config. + """ + + def __init__(self, model_dir, model_config: Dict[str, Any], *args, + **kwargs): + super().__init__(model_dir, model_config, *args, **kwargs) + self.model_config = model_config + self.embed_dim = self.model_config['embed_dim'] + self.m_channels = self.model_config['channels'] + self.other_config = kwargs + self.feature_dim = 80 + self.device = create_device(self.other_config['device']) + + self.embedding_model = Res2Net( + embedding_size=self.embed_dim, m_channels=self.m_channels) + + pretrained_model_name = kwargs['pretrained_model'] + self.__load_check_point(pretrained_model_name) + + self.embedding_model.to(self.device) + self.embedding_model.eval() + + def forward(self, audio): + if isinstance(audio, np.ndarray): + audio = torch.from_numpy(audio) + if len(audio.shape) == 1: + audio = audio.unsqueeze(0) + assert len( + audio.shape + ) == 2, 'modelscope error: the shape of input audio to model needs to be [N, T]' + # audio shape: [N, T] + feature = self.__extract_feature(audio) + embedding = self.embedding_model(feature.to(self.device)) + + return embedding.detach().cpu() + + def __extract_feature(self, audio): + feature = Kaldi.fbank(audio, num_mel_bins=self.feature_dim) + feature = feature - feature.mean(dim=0, keepdim=True) + feature = feature.unsqueeze(0) + return feature + + def __load_check_point(self, pretrained_model_name, device=None): + if not device: + device = torch.device('cpu') + self.embedding_model.load_state_dict( + torch.load( + os.path.join(self.model_dir, pretrained_model_name), + map_location=device), + strict=True) diff --git a/modelscope/models/audio/sv/ResNet.py b/modelscope/models/audio/sv/ResNet.py new file mode 100644 index 00000000..94d303b5 --- /dev/null +++ b/modelscope/models/audio/sv/ResNet.py @@ -0,0 +1,186 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +""" ResNet implementation is adapted from https://github.com/wenet-e2e/wespeaker. + ResNet, or Residual Neural Network, is notable for its optimization ease + and depth-induced accuracy gains. It utilizes skip connections within its residual + blocks to counteract the vanishing gradient problem in deep networks. + Reference: Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun + Deep Residual Learning for Image Recognition. arXiv:1512.03385 +""" +import math +import os +from typing import Any, Dict, Union + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchaudio.compliance.kaldi as Kaldi + +import modelscope.models.audio.sv.pooling_layers as pooling_layers +from modelscope.metainfo import Models +from modelscope.models import MODELS, TorchModel +from modelscope.utils.constant import Tasks +from modelscope.utils.device import create_device + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, in_planes, planes, stride=1): + super(BasicBlock, self).__init__() + self.conv1 = nn.Conv2d( + in_planes, + planes, + kernel_size=3, + stride=stride, + padding=1, + bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d( + planes, planes, kernel_size=3, stride=1, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion * planes: + self.shortcut = nn.Sequential( + nn.Conv2d( + in_planes, + self.expansion * planes, + kernel_size=1, + stride=stride, + bias=False), nn.BatchNorm2d(self.expansion * planes)) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.bn2(self.conv2(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + +class ResNet(nn.Module): + + def __init__(self, + block=BasicBlock, + num_blocks=[3, 4, 6, 3], + m_channels=32, + feat_dim=80, + embedding_size=128, + pooling_func='TSTP', + two_emb_layer=True): + super(ResNet, self).__init__() + self.in_planes = m_channels + self.feat_dim = feat_dim + self.embedding_size = embedding_size + self.stats_dim = int(feat_dim / 8) * m_channels * 8 + self.two_emb_layer = two_emb_layer + + self.conv1 = nn.Conv2d( + 1, m_channels, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(m_channels) + + self.layer1 = self._make_layer( + block, m_channels, num_blocks[0], stride=1) + self.layer2 = self._make_layer( + block, m_channels * 2, num_blocks[1], stride=2) + self.layer3 = self._make_layer( + block, m_channels * 4, num_blocks[2], stride=2) + self.layer4 = self._make_layer( + block, m_channels * 8, num_blocks[3], stride=2) + + self.n_stats = 1 if pooling_func == 'TAP' or pooling_func == 'TSDP' else 2 + self.pool = getattr(pooling_layers, pooling_func)( + in_dim=self.stats_dim * block.expansion) + self.seg_1 = nn.Linear(self.stats_dim * block.expansion * self.n_stats, + embedding_size) + if self.two_emb_layer: + self.seg_bn_1 = nn.BatchNorm1d(embedding_size, affine=False) + self.seg_2 = nn.Linear(embedding_size, embedding_size) + else: + self.seg_bn_1 = nn.Identity() + self.seg_2 = nn.Identity() + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1] * (num_blocks - 1) + layers = [] + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + return nn.Sequential(*layers) + + def forward(self, x): + x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T) + x = x.unsqueeze_(1) + out = F.relu(self.bn1(self.conv1(x))) + out1 = self.layer1(out) + out2 = self.layer2(out1) + out3 = self.layer3(out2) + out = self.layer4(out3) + stats = self.pool(out) + + embed_a = self.seg_1(stats) + if self.two_emb_layer: + out = F.relu(embed_a) + out = self.seg_bn_1(out) + embed_b = self.seg_2(out) + return embed_b + else: + return embed_a + + +@MODELS.register_module( + Tasks.speaker_verification, module_name=Models.resnet_sv) +class SpeakerVerificationResNet(TorchModel): + r""" + Args: + model_dir: A model dir. + model_config: The model config. + """ + + def __init__(self, model_dir, model_config: Dict[str, Any], *args, + **kwargs): + super().__init__(model_dir, model_config, *args, **kwargs) + self.model_config = model_config + self.embed_dim = self.model_config['embed_dim'] + self.m_channels = self.model_config['channels'] + self.other_config = kwargs + self.feature_dim = 80 + self.device = create_device(self.other_config['device']) + + self.embedding_model = ResNet( + embedding_size=self.embed_dim, m_channels=self.m_channels) + + pretrained_model_name = kwargs['pretrained_model'] + self.__load_check_point(pretrained_model_name) + + self.embedding_model.to(self.device) + self.embedding_model.eval() + + def forward(self, audio): + if isinstance(audio, np.ndarray): + audio = torch.from_numpy(audio) + if len(audio.shape) == 1: + audio = audio.unsqueeze(0) + assert len( + audio.shape + ) == 2, 'modelscope error: the shape of input audio to model needs to be [N, T]' + # audio shape: [N, T] + feature = self.__extract_feature(audio) + embedding = self.embedding_model(feature.to(self.device)) + + return embedding.detach().cpu() + + def __extract_feature(self, audio): + feature = Kaldi.fbank(audio, num_mel_bins=self.feature_dim) + feature = feature - feature.mean(dim=0, keepdim=True) + feature = feature.unsqueeze(0) + return feature + + def __load_check_point(self, pretrained_model_name, device=None): + if not device: + device = torch.device('cpu') + self.embedding_model.load_state_dict( + torch.load( + os.path.join(self.model_dir, pretrained_model_name), + map_location=device), + strict=True) diff --git a/modelscope/pipelines/audio/speaker_verification_res2net_pipeline.py b/modelscope/pipelines/audio/speaker_verification_res2net_pipeline.py new file mode 100644 index 00000000..d64f371e --- /dev/null +++ b/modelscope/pipelines/audio/speaker_verification_res2net_pipeline.py @@ -0,0 +1,159 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import io +from typing import Any, Dict, List, Union + +import numpy as np +import soundfile as sf +import torch +import torchaudio + +from modelscope.fileio import File +from modelscope.metainfo import Pipelines +from modelscope.outputs import OutputKeys +from modelscope.pipelines.base import InputModel, Pipeline +from modelscope.pipelines.builder import PIPELINES +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +@PIPELINES.register_module( + Tasks.speaker_verification, + module_name=Pipelines.speaker_verification_res2net) +class Res2Net_Pipeline(Pipeline): + """Speaker Verification Inference Pipeline + use `model` to create a Speaker Verification pipeline. + + Args: + model (SpeakerVerificationPipeline): A model instance, or a model local dir, or a model id in the model hub. + kwargs (dict, `optional`): + Extra kwargs passed into the pipeline's constructor. + Example: + >>> from modelscope.pipelines import pipeline + >>> from modelscope.utils.constant import Tasks + >>> p = pipeline( + >>> task=Tasks.speaker_verification, model='iic/speech_res2net_sv_zh-cn_3dspeaker_16k') + >>> print(p([audio_1, audio_2])) + + """ + + def __init__(self, model: InputModel, **kwargs): + """use `model` to create a speaker verification pipeline for prediction + Args: + model (str): a valid offical model id + """ + super().__init__(model=model, **kwargs) + self.model_config = self.model.model_config + self.config = self.model.other_config + self.thr = self.config['yesOrno_thr'] + self.save_dict = {} + + def __call__(self, + in_audios: Union[np.ndarray, list], + save_dir: str = None, + output_emb: bool = False, + thr: float = None): + if thr is not None: + self.thr = thr + if self.thr < -1 or self.thr > 1: + raise ValueError( + 'modelscope error: the thr value should be in [-1, 1], but found to be %f.' + % self.thr) + wavs = self.preprocess(in_audios) + embs = self.forward(wavs) + outputs = self.postprocess(embs, in_audios, save_dir) + if output_emb: + self.save_dict['outputs'] = outputs + self.save_dict['embs'] = embs.numpy() + return self.save_dict + else: + return outputs + + def forward(self, inputs: list): + embs = [] + for x in inputs: + embs.append(self.model(x)) + embs = torch.cat(embs) + return embs + + def postprocess(self, + inputs: torch.Tensor, + in_audios: Union[np.ndarray, list], + save_dir=None): + if isinstance(in_audios[0], str) and save_dir is not None: + # save the embeddings + os.makedirs(save_dir, exist_ok=True) + for i, p in enumerate(in_audios): + save_path = os.path.join( + save_dir, '%s.npy' % + (os.path.basename(p).rsplit('.', 1)[0])) + np.save(save_path, inputs[i].numpy()) + + if len(inputs) == 2: + # compute the score + score = self.compute_cos_similarity(inputs[0], inputs[1]) + score = round(score, 5) + if score >= self.thr: + ans = 'yes' + else: + ans = 'no' + output = {OutputKeys.SCORE: score, OutputKeys.TEXT: ans} + else: + output = {OutputKeys.TEXT: 'No similarity score output'} + + return output + + def preprocess(self, inputs: Union[np.ndarray, list]): + output = [] + for i in range(len(inputs)): + if isinstance(inputs[i], str): + file_bytes = File.read(inputs[i]) + data, fs = sf.read(io.BytesIO(file_bytes), dtype='float32') + if len(data.shape) == 2: + data = data[:, 0] + data = torch.from_numpy(data).unsqueeze(0) + if fs != self.model_config['sample_rate']: + logger.warning( + 'The sample rate of audio is not %d, resample it.' + % self.model_config['sample_rate']) + data, fs = torchaudio.sox_effects.apply_effects_tensor( + data, + fs, + effects=[[ + 'rate', + str(self.model_config['sample_rate']) + ]]) + data = data.squeeze(0) + elif isinstance(inputs[i], np.ndarray): + assert len( + inputs[i].shape + ) == 1, 'modelscope error: Input array should be [N, T]' + data = inputs[i] + if data.dtype in ['int16', 'int32', 'int64']: + data = (data / (1 << 15)).astype('float32') + else: + data = data.astype('float32') + data = torch.from_numpy(data) + else: + raise ValueError( + 'modelscope error: The input type is restricted to audio address and nump array.' + ) + output.append(data) + return output + + def compute_cos_similarity(self, emb1: Union[np.ndarray, torch.Tensor], + emb2: Union[np.ndarray, torch.Tensor]) -> float: + if isinstance(emb1, np.ndarray): + emb1 = torch.from_numpy(emb1) + if isinstance(emb2, np.ndarray): + emb2 = torch.from_numpy(emb2) + if len(emb1.shape): + emb1 = emb1.unsqueeze(0) + if len(emb2.shape): + emb2 = emb2.unsqueeze(0) + assert len(emb1.shape) == 2 and len(emb2.shape) == 2 + cos = torch.nn.CosineSimilarity(dim=1, eps=1e-6) + cosine = cos(emb1, emb2) + return cosine.item() diff --git a/modelscope/pipelines/audio/speaker_verification_resnet_pipeline.py b/modelscope/pipelines/audio/speaker_verification_resnet_pipeline.py new file mode 100644 index 00000000..54cafb28 --- /dev/null +++ b/modelscope/pipelines/audio/speaker_verification_resnet_pipeline.py @@ -0,0 +1,159 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import io +from typing import Any, Dict, List, Union + +import numpy as np +import soundfile as sf +import torch +import torchaudio + +from modelscope.fileio import File +from modelscope.metainfo import Pipelines +from modelscope.outputs import OutputKeys +from modelscope.pipelines.base import InputModel, Pipeline +from modelscope.pipelines.builder import PIPELINES +from modelscope.utils.constant import Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + + +@PIPELINES.register_module( + Tasks.speaker_verification, + module_name=Pipelines.speaker_verification_resnet) +class ResNet_Pipeline(Pipeline): + """Speaker Verification Inference Pipeline + use `model` to create a Speaker Verification pipeline. + + Args: + model (SpeakerVerificationPipeline): A model instance, or a model local dir, or a model id in the model hub. + kwargs (dict, `optional`): + Extra kwargs passed into the pipeline's constructor. + Example: + >>> from modelscope.pipelines import pipeline + >>> from modelscope.utils.constant import Tasks + >>> p = pipeline( + >>> task=Tasks.speaker_verification, model='iic/speech_resnet34_sv_zh-cn_3dspeaker_16k') + >>> print(p([audio_1, audio_2])) + + """ + + def __init__(self, model: InputModel, **kwargs): + """use `model` to create a speaker verification pipeline for prediction + Args: + model (str): a valid offical model id + """ + super().__init__(model=model, **kwargs) + self.model_config = self.model.model_config + self.config = self.model.other_config + self.thr = self.config['yesOrno_thr'] + self.save_dict = {} + + def __call__(self, + in_audios: Union[np.ndarray, list], + save_dir: str = None, + output_emb: bool = False, + thr: float = None): + if thr is not None: + self.thr = thr + if self.thr < -1 or self.thr > 1: + raise ValueError( + 'modelscope error: the thr value should be in [-1, 1], but found to be %f.' + % self.thr) + wavs = self.preprocess(in_audios) + embs = self.forward(wavs) + outputs = self.postprocess(embs, in_audios, save_dir) + if output_emb: + self.save_dict['outputs'] = outputs + self.save_dict['embs'] = embs.numpy() + return self.save_dict + else: + return outputs + + def forward(self, inputs: list): + embs = [] + for x in inputs: + embs.append(self.model(x)) + embs = torch.cat(embs) + return embs + + def postprocess(self, + inputs: torch.Tensor, + in_audios: Union[np.ndarray, list], + save_dir=None): + if isinstance(in_audios[0], str) and save_dir is not None: + # save the embeddings + os.makedirs(save_dir, exist_ok=True) + for i, p in enumerate(in_audios): + save_path = os.path.join( + save_dir, '%s.npy' % + (os.path.basename(p).rsplit('.', 1)[0])) + np.save(save_path, inputs[i].numpy()) + + if len(inputs) == 2: + # compute the score + score = self.compute_cos_similarity(inputs[0], inputs[1]) + score = round(score, 5) + if score >= self.thr: + ans = 'yes' + else: + ans = 'no' + output = {OutputKeys.SCORE: score, OutputKeys.TEXT: ans} + else: + output = {OutputKeys.TEXT: 'No similarity score output'} + + return output + + def preprocess(self, inputs: Union[np.ndarray, list]): + output = [] + for i in range(len(inputs)): + if isinstance(inputs[i], str): + file_bytes = File.read(inputs[i]) + data, fs = sf.read(io.BytesIO(file_bytes), dtype='float32') + if len(data.shape) == 2: + data = data[:, 0] + data = torch.from_numpy(data).unsqueeze(0) + if fs != self.model_config['sample_rate']: + logger.warning( + 'The sample rate of audio is not %d, resample it.' + % self.model_config['sample_rate']) + data, fs = torchaudio.sox_effects.apply_effects_tensor( + data, + fs, + effects=[[ + 'rate', + str(self.model_config['sample_rate']) + ]]) + data = data.squeeze(0) + elif isinstance(inputs[i], np.ndarray): + assert len( + inputs[i].shape + ) == 1, 'modelscope error: Input array should be [N, T]' + data = inputs[i] + if data.dtype in ['int16', 'int32', 'int64']: + data = (data / (1 << 15)).astype('float32') + else: + data = data.astype('float32') + data = torch.from_numpy(data) + else: + raise ValueError( + 'modelscope error: The input type is restricted to audio address and nump array.' + ) + output.append(data) + return output + + def compute_cos_similarity(self, emb1: Union[np.ndarray, torch.Tensor], + emb2: Union[np.ndarray, torch.Tensor]) -> float: + if isinstance(emb1, np.ndarray): + emb1 = torch.from_numpy(emb1) + if isinstance(emb2, np.ndarray): + emb2 = torch.from_numpy(emb2) + if len(emb1.shape): + emb1 = emb1.unsqueeze(0) + if len(emb2.shape): + emb2 = emb2.unsqueeze(0) + assert len(emb1.shape) == 2 and len(emb2.shape) == 2 + cos = torch.nn.CosineSimilarity(dim=1, eps=1e-6) + cosine = cos(emb1, emb2) + return cosine.item() diff --git a/tests/pipelines/test_speaker_verification.py b/tests/pipelines/test_speaker_verification.py index c5fe0004..42ea3c83 100644 --- a/tests/pipelines/test_speaker_verification.py +++ b/tests/pipelines/test_speaker_verification.py @@ -34,11 +34,10 @@ class SpeakerVerificationTest(unittest.TestCase): rdino_3dspeaker_16k_model_id = 'damo/speech_rdino_ecapa_tdnn_sv_zh-cn_3dspeaker_16k' eres2net_base_3dspeaker_16k_model_id = 'damo/speech_eres2net_base_sv_zh-cn_3dspeaker_16k' eres2net_large_3dspeaker_16k_model_id = 'damo/speech_eres2net_large_sv_zh-cn_3dspeaker_16k' + resnet_3dspeaker_16k_model_id = 'iic/speech_resnet34_sv_zh-cn_3dspeaker_16k' + res2net_3dspeaker_16k_model_id = 'iic/speech_res2net_sv_zh-cn_3dspeaker_16k' lre_eres2net_large_five_lang_8k_model_id = 'damo/speech_eres2net_large_five_lre_8k' - def setUp(self) -> None: - self.task = Tasks.speaker_verification - def run_pipeline(self, model_id: str, audios: Union[List[str], str], @@ -46,6 +45,8 @@ class SpeakerVerificationTest(unittest.TestCase): model_revision=None) -> Dict[str, Any]: if task is not None: self.task = task + else: + self.task = Tasks.speaker_verification p = pipeline( task=self.task, model=model_id, model_revision=model_revision) result = p(audios) @@ -104,6 +105,26 @@ class SpeakerVerificationTest(unittest.TestCase): print(result) self.assertTrue(OutputKeys.SCORE in result) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_speaker_verification_resnet_3dspeaker_16k(self): + logger.info('Run speaker verification for resnet_3dspeaker_16k model') + result = self.run_pipeline( + model_id=self.resnet_3dspeaker_16k_model_id, + audios=[SPEAKER1_A_EN_16K_WAV, SPEAKER1_B_EN_16K_WAV], + model_revision='v1.0.0') + print(result) + self.assertTrue(OutputKeys.SCORE in result) + + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_speaker_verification_res2net_3dspeaker_16k(self): + logger.info('Run speaker verification for res2net_3dspeaker_16k model') + result = self.run_pipeline( + model_id=self.res2net_3dspeaker_16k_model_id, + audios=[SPEAKER1_A_EN_16K_WAV, SPEAKER1_B_EN_16K_WAV], + model_revision='v1.0.0') + print(result) + self.assertTrue(OutputKeys.SCORE in result) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_run_with_speaker_verification_rdino_3dspeaker_16k(self): logger.info('Run speaker verification for rdino_3dspeaker_16k model')