From 41da5d8e90a7d7d801de775f733042586a3baffe Mon Sep 17 00:00:00 2001 From: "tongmu.wh" Date: Mon, 10 Apr 2023 16:27:13 +0800 Subject: [PATCH] add speaker model cam++ for speaker verification task damo/speech_campplus_sv_en_voxceleb_16k MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 说话人识别的新模型，模型库已创建在https://modelscope.cn/models/damo/speech_campplus_sv_en_voxceleb_16k/summary Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/12180950 --- modelscope/metainfo.py | 1 + modelscope/models/audio/sv/DTDNN.py | 187 +++++++++++++ modelscope/models/audio/sv/DTDNN_layers.py | 266 +++++++++++++++++++ modelscope/pipelines/audio/__init__.py | 1 + tests/pipelines/test_speaker_verification.py | 11 + 5 files changed, 466 insertions(+) create mode 100644 modelscope/models/audio/sv/DTDNN.py create mode 100644 modelscope/models/audio/sv/DTDNN_layers.py diff --git a/modelscope/metainfo.py b/modelscope/metainfo.py index 0314cfe0..72663435 100644 --- a/modelscope/metainfo.py +++ b/modelscope/metainfo.py @@ -182,6 +182,7 @@ class Models(object): generic_punc = 'generic-punc' generic_sv = 'generic-sv' ecapa_tdnn_sv = 'ecapa-tdnn-sv' + campplus_sv = 'cam++-sv' generic_lm = 'generic-lm' # multi-modal models diff --git a/modelscope/models/audio/sv/DTDNN.py b/modelscope/models/audio/sv/DTDNN.py new file mode 100644 index 00000000..d9e21ce8 --- /dev/null +++ b/modelscope/models/audio/sv/DTDNN.py @@ -0,0 +1,187 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os +from collections import OrderedDict +from typing import Any, Dict, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchaudio.compliance.kaldi as Kaldi + +from modelscope.metainfo import Models +from modelscope.models import MODELS, TorchModel +from modelscope.models.audio.sv.DTDNN_layers import (BasicResBlock, + CAMDenseTDNNBlock, + DenseLayer, StatsPool, + TDNNLayer, TransitLayer, + get_nonlinear) +from modelscope.utils.constant import Tasks + + +class FCM(nn.Module): + + def __init__(self, + block=BasicResBlock, + num_blocks=[2, 2], + m_channels=32, + feat_dim=80): + super(FCM, self).__init__() + self.in_planes = m_channels + self.conv1 = nn.Conv2d( + 1, m_channels, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(m_channels) + + self.layer1 = self._make_layer( + block, m_channels, num_blocks[0], stride=2) + self.layer2 = self._make_layer( + block, m_channels, num_blocks[0], stride=2) + + self.conv2 = nn.Conv2d( + m_channels, + m_channels, + kernel_size=3, + stride=(2, 1), + padding=1, + bias=False) + self.bn2 = nn.BatchNorm2d(m_channels) + self.out_channels = m_channels * (feat_dim // 8) + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1] * (num_blocks - 1) + layers = [] + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + return nn.Sequential(*layers) + + def forward(self, x): + x = x.unsqueeze(1) + out = F.relu(self.bn1(self.conv1(x))) + out = self.layer1(out) + out = self.layer2(out) + out = F.relu(self.bn2(self.conv2(out))) + + shape = out.shape + out = out.reshape(shape[0], shape[1] * shape[2], shape[3]) + return out + + +class CAMPPlus(nn.Module): + + def __init__(self, + feat_dim=80, + embedding_size=512, + growth_rate=32, + bn_size=4, + init_channels=128, + config_str='batchnorm-relu', + memory_efficient=True): + super(CAMPPlus, self).__init__() + + self.head = FCM(feat_dim=feat_dim) + channels = self.head.out_channels + + self.xvector = nn.Sequential( + OrderedDict([ + ('tdnn', + TDNNLayer( + channels, + init_channels, + 5, + stride=2, + dilation=1, + padding=-1, + config_str=config_str)), + ])) + channels = init_channels + for i, (num_layers, kernel_size, dilation) in enumerate( + zip((12, 24, 16), (3, 3, 3), (1, 2, 2))): + block = CAMDenseTDNNBlock( + num_layers=num_layers, + in_channels=channels, + out_channels=growth_rate, + bn_channels=bn_size * growth_rate, + kernel_size=kernel_size, + dilation=dilation, + config_str=config_str, + memory_efficient=memory_efficient) + self.xvector.add_module('block%d' % (i + 1), block) + channels = channels + num_layers * growth_rate + self.xvector.add_module( + 'transit%d' % (i + 1), + TransitLayer( + channels, channels // 2, bias=False, + config_str=config_str)) + channels //= 2 + + self.xvector.add_module('out_nonlinear', + get_nonlinear(config_str, channels)) + + self.xvector.add_module('stats', StatsPool()) + self.xvector.add_module( + 'dense', + DenseLayer(channels * 2, embedding_size, config_str='batchnorm_')) + + for m in self.modules(): + if isinstance(m, (nn.Conv1d, nn.Linear)): + nn.init.kaiming_normal_(m.weight.data) + if m.bias is not None: + nn.init.zeros_(m.bias) + + def forward(self, x): + x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T) + x = self.head(x) + x = self.xvector(x) + return x + + +@MODELS.register_module( + Tasks.speaker_verification, module_name=Models.campplus_sv) +class SpeakerVerificationCAMPPlus(TorchModel): + r"""A fast and efficient speaker embedding model, using a 2-dimensional convolution residual network as the head + and a densely connected time delay neural network as the backbone. + Args: + model_dir: A model dir. + model_config: The model config. + """ + + def __init__(self, model_dir, model_config: Dict[str, Any], *args, + **kwargs): + super().__init__(model_dir, model_config, *args, **kwargs) + self.model_config = model_config + self.other_config = kwargs + + self.feature_dim = self.model_config['fbank_dim'] + self.emb_size = self.model_config['emb_size'] + + self.embedding_model = CAMPPlus(self.feature_dim, self.emb_size) + + pretrained_model_name = kwargs['pretrained_model'] + self.__load_check_point(pretrained_model_name) + + self.embedding_model.eval() + + def forward(self, audio): + assert len(audio.shape) == 2 and audio.shape[ + 0] == 1, 'modelscope error: the shape of input audio to model needs to be [1, T]' + # audio shape: [1, T] + feature = self.__extract_feature(audio) + embedding = self.embedding_model(feature) + + return embedding + + def __extract_feature(self, audio): + feature = Kaldi.fbank(audio, num_mel_bins=self.feature_dim) + feature = feature - feature.mean(dim=0, keepdim=True) + feature = feature.unsqueeze(0) + return feature + + def __load_check_point(self, pretrained_model_name, device=None): + if not device: + device = torch.device('cpu') + self.embedding_model.load_state_dict( + torch.load( + os.path.join(self.model_dir, pretrained_model_name), + map_location=device), + strict=True) diff --git a/modelscope/models/audio/sv/DTDNN_layers.py b/modelscope/models/audio/sv/DTDNN_layers.py new file mode 100644 index 00000000..21e34621 --- /dev/null +++ b/modelscope/models/audio/sv/DTDNN_layers.py @@ -0,0 +1,266 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +""" Some implementations are adapted from https://github.com/yuyq96/D-TDNN +""" + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint as cp +from torch import nn + + +def get_nonlinear(config_str, channels): + nonlinear = nn.Sequential() + for name in config_str.split('-'): + if name == 'relu': + nonlinear.add_module('relu', nn.ReLU(inplace=True)) + elif name == 'prelu': + nonlinear.add_module('prelu', nn.PReLU(channels)) + elif name == 'batchnorm': + nonlinear.add_module('batchnorm', nn.BatchNorm1d(channels)) + elif name == 'batchnorm_': + nonlinear.add_module('batchnorm', + nn.BatchNorm1d(channels, affine=False)) + else: + raise ValueError('Unexpected module ({}).'.format(name)) + return nonlinear + + +def statistics_pooling(x, dim=-1, keepdim=False, unbiased=True, eps=1e-2): + mean = x.mean(dim=dim) + std = x.std(dim=dim, unbiased=unbiased) + stats = torch.cat([mean, std], dim=-1) + if keepdim: + stats = stats.unsqueeze(dim=dim) + return stats + + +class StatsPool(nn.Module): + + def forward(self, x): + return statistics_pooling(x) + + +class TDNNLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + bias=False, + config_str='batchnorm-relu'): + super(TDNNLayer, self).__init__() + if padding < 0: + assert kernel_size % 2 == 1, 'Expect equal paddings, but got even kernel size ({})'.format( + kernel_size) + padding = (kernel_size - 1) // 2 * dilation + self.linear = nn.Conv1d( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias) + self.nonlinear = get_nonlinear(config_str, out_channels) + + def forward(self, x): + x = self.linear(x) + x = self.nonlinear(x) + return x + + +class CAMLayer(nn.Module): + + def __init__(self, + bn_channels, + out_channels, + kernel_size, + stride, + padding, + dilation, + bias, + reduction=2): + super(CAMLayer, self).__init__() + self.linear_local = nn.Conv1d( + bn_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias) + self.linear1 = nn.Conv1d(bn_channels, bn_channels // reduction, 1) + self.relu = nn.ReLU(inplace=True) + self.linear2 = nn.Conv1d(bn_channels // reduction, out_channels, 1) + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + y = self.linear_local(x) + context = x.mean(-1, keepdim=True) + self.seg_pooling(x) + context = self.relu(self.linear1(context)) + m = self.sigmoid(self.linear2(context)) + return y * m + + def seg_pooling(self, x, seg_len=100, stype='avg'): + if stype == 'avg': + seg = F.avg_pool1d( + x, kernel_size=seg_len, stride=seg_len, ceil_mode=True) + elif stype == 'max': + seg = F.max_pool1d( + x, kernel_size=seg_len, stride=seg_len, ceil_mode=True) + else: + raise ValueError('Wrong segment pooling type.') + shape = seg.shape + seg = seg.unsqueeze(-1).expand(*shape, + seg_len).reshape(*shape[:-1], -1) + seg = seg[..., :x.shape[-1]] + return seg + + +class CAMDenseTDNNLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + bn_channels, + kernel_size, + stride=1, + dilation=1, + bias=False, + config_str='batchnorm-relu', + memory_efficient=False): + super(CAMDenseTDNNLayer, self).__init__() + assert kernel_size % 2 == 1, 'Expect equal paddings, but got even kernel size ({})'.format( + kernel_size) + padding = (kernel_size - 1) // 2 * dilation + self.memory_efficient = memory_efficient + self.nonlinear1 = get_nonlinear(config_str, in_channels) + self.linear1 = nn.Conv1d(in_channels, bn_channels, 1, bias=False) + self.nonlinear2 = get_nonlinear(config_str, bn_channels) + self.cam_layer = CAMLayer( + bn_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + bias=bias) + + def bn_function(self, x): + return self.linear1(self.nonlinear1(x)) + + def forward(self, x): + if self.training and self.memory_efficient: + x = cp.checkpoint(self.bn_function, x) + else: + x = self.bn_function(x) + x = self.cam_layer(self.nonlinear2(x)) + return x + + +class CAMDenseTDNNBlock(nn.ModuleList): + + def __init__(self, + num_layers, + in_channels, + out_channels, + bn_channels, + kernel_size, + stride=1, + dilation=1, + bias=False, + config_str='batchnorm-relu', + memory_efficient=False): + super(CAMDenseTDNNBlock, self).__init__() + for i in range(num_layers): + layer = CAMDenseTDNNLayer( + in_channels=in_channels + i * out_channels, + out_channels=out_channels, + bn_channels=bn_channels, + kernel_size=kernel_size, + stride=stride, + dilation=dilation, + bias=bias, + config_str=config_str, + memory_efficient=memory_efficient) + self.add_module('tdnnd%d' % (i + 1), layer) + + def forward(self, x): + for layer in self: + x = torch.cat([x, layer(x)], dim=1) + return x + + +class TransitLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + bias=True, + config_str='batchnorm-relu'): + super(TransitLayer, self).__init__() + self.nonlinear = get_nonlinear(config_str, in_channels) + self.linear = nn.Conv1d(in_channels, out_channels, 1, bias=bias) + + def forward(self, x): + x = self.nonlinear(x) + x = self.linear(x) + return x + + +class DenseLayer(nn.Module): + + def __init__(self, + in_channels, + out_channels, + bias=False, + config_str='batchnorm-relu'): + super(DenseLayer, self).__init__() + self.linear = nn.Conv1d(in_channels, out_channels, 1, bias=bias) + self.nonlinear = get_nonlinear(config_str, out_channels) + + def forward(self, x): + if len(x.shape) == 2: + x = self.linear(x.unsqueeze(dim=-1)).squeeze(dim=-1) + else: + x = self.linear(x) + x = self.nonlinear(x) + return x + + +class BasicResBlock(nn.Module): + expansion = 1 + + def __init__(self, in_planes, planes, stride=1): + super(BasicResBlock, self).__init__() + self.conv1 = nn.Conv2d( + in_planes, + planes, + kernel_size=3, + stride=(stride, 1), + padding=1, + bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d( + planes, planes, kernel_size=3, stride=1, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion * planes: + self.shortcut = nn.Sequential( + nn.Conv2d( + in_planes, + self.expansion * planes, + kernel_size=1, + stride=(stride, 1), + bias=False), nn.BatchNorm2d(self.expansion * planes)) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.bn2(self.conv2(out)) + out += self.shortcut(x) + out = F.relu(out) + return out diff --git a/modelscope/pipelines/audio/__init__.py b/modelscope/pipelines/audio/__init__.py index 18e8b8b3..14453b51 100644 --- a/modelscope/pipelines/audio/__init__.py +++ b/modelscope/pipelines/audio/__init__.py @@ -21,6 +21,7 @@ else: 'kws_kwsbp_pipeline': ['KeyWordSpottingKwsbpPipeline'], 'linear_aec_pipeline': ['LinearAECPipeline'], 'text_to_speech_pipeline': ['TextToSpeechSambertHifiganPipeline'], + 'itn_inference_pipeline': ['InverseTextProcessingPipeline'], 'inverse_text_processing_pipeline': ['InverseTextProcessingPipeline'], 'speaker_verification_pipeline': ['SpeakerVerificationPipeline'] } diff --git a/tests/pipelines/test_speaker_verification.py b/tests/pipelines/test_speaker_verification.py index addb9058..83d8aff3 100644 --- a/tests/pipelines/test_speaker_verification.py +++ b/tests/pipelines/test_speaker_verification.py @@ -20,6 +20,7 @@ SPEAKER2_A_EN_16K_WAV = 'data/test/audios/speaker2_a_en_16k.wav' class SpeakerVerificationTest(unittest.TestCase, DemoCompatibilityCheck): ecapatdnn_voxceleb_16k_model_id = 'damo/speech_ecapa-tdnn_sv_en_voxceleb_16k' + campplus_voxceleb_16k_model_id = 'damo/speech_campplus_sv_en_voxceleb_16k' def setUp(self) -> None: self.task = Tasks.speaker_verification @@ -40,6 +41,16 @@ class SpeakerVerificationTest(unittest.TestCase, DemoCompatibilityCheck): print(result) self.assertTrue(OutputKeys.SCORE in result) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_with_speaker_verification_campplus_voxceleb_16k(self): + logger.info('Run speaker verification for campplus_voxceleb_16k model') + + result = self.run_pipeline( + model_id=self.campplus_voxceleb_16k_model_id, + audios=[SPEAKER1_A_EN_16K_WAV, SPEAKER2_A_EN_16K_WAV]) + print(result) + self.assertTrue(OutputKeys.SCORE in result) + @unittest.skip('demo compatibility test is only enabled on a needed-basis') def test_demo_compatibility(self): self.compatibility_check()