diff --git a/modelscope/metainfo.py b/modelscope/metainfo.py index 2af135b7..33f1a7ca 100644 --- a/modelscope/metainfo.py +++ b/modelscope/metainfo.py @@ -416,6 +416,7 @@ class Pipelines(object): itn_inference = 'itn-inference' punc_inference = 'punc-inference' sv_inference = 'sv-inference' + vad_inference = 'vad-inference' speaker_verification = 'speaker-verification' lm_inference = 'language-model' diff --git a/modelscope/models/audio/asr/generic_automatic_speech_recognition.py b/modelscope/models/audio/asr/generic_automatic_speech_recognition.py index 4858cd5a..8e73b738 100644 --- a/modelscope/models/audio/asr/generic_automatic_speech_recognition.py +++ b/modelscope/models/audio/asr/generic_automatic_speech_recognition.py @@ -13,6 +13,8 @@ __all__ = ['GenericAutomaticSpeechRecognition'] @MODELS.register_module( Tasks.auto_speech_recognition, module_name=Models.generic_asr) +@MODELS.register_module( + Tasks.voice_activity_detection, module_name=Models.generic_asr) @MODELS.register_module(Tasks.language_model, module_name=Models.generic_asr) class GenericAutomaticSpeechRecognition(Model): diff --git a/modelscope/msdatasets/audio/__init__.py b/modelscope/msdatasets/audio/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modelscope/msdatasets/audio/asr_dataset.py b/modelscope/msdatasets/audio/asr_dataset.py new file mode 100644 index 00000000..c0696615 --- /dev/null +++ b/modelscope/msdatasets/audio/asr_dataset.py @@ -0,0 +1,48 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import os + +from modelscope.msdatasets.ms_dataset import MsDataset + + +class ASRDataset(MsDataset): + """ASR dataset for speech recognition. + support load dataset from msdataset hub or local data_dir (including wav.scp and text) + For more details, please refer to + https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/datasets/ms_dataset.py. + """ + + @classmethod + def load_core(cls, data_dir, data_set): + wav_file = os.path.join(data_dir, data_set, 'wav.scp') + text_file = os.path.join(data_dir, data_set, 'text') + with open(wav_file) as f: + wav_lines = f.readlines() + with open(text_file) as f: + text_lines = f.readlines() + data_list = [] + for wav_line, text_line in zip(wav_lines, text_lines): + item = {} + item['Audio:FILE'] = wav_line.strip().split()[-1] + item['Text:LABEL'] = ' '.join(text_line.strip().split()[1:]) + data_list.append(item) + return data_list + + @classmethod + def load(cls, + dataset_name, + namespace='speech_asr', + train_set='train', + dev_set='validation'): + if os.path.exists(dataset_name): + data_dir = dataset_name + ds_dict = {} + ds_dict['train'] = cls.load_core(data_dir, train_set) + ds_dict['validation'] = cls.load_core(data_dir, dev_set) + ds_dict['raw_data_dir'] = data_dir + return ds_dict + else: + from modelscope.msdatasets import MsDataset + ds_dict = MsDataset.load( + dataset_name=dataset_name, namespace=namespace) + return ds_dict diff --git a/modelscope/pipelines/audio/asr_inference_pipeline.py b/modelscope/pipelines/audio/asr_inference_pipeline.py index 8cf9e354..e259bc44 100644 --- a/modelscope/pipelines/audio/asr_inference_pipeline.py +++ b/modelscope/pipelines/audio/asr_inference_pipeline.py @@ -1,7 +1,8 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import os -from typing import Any, Dict, List, Sequence, Tuple, Union +from typing import Any, Dict, List, Optional, Sequence, Tuple, Union +import json import yaml from modelscope.metainfo import Pipelines @@ -13,7 +14,8 @@ from modelscope.preprocessors import WavToScp from modelscope.utils.audio.audio_utils import (extract_pcm_from_wav, generate_scp_from_url, load_bytes_from_url) -from modelscope.utils.constant import Frameworks, Tasks +from modelscope.utils.constant import Frameworks, ModelFile, Tasks +from modelscope.utils.hub import snapshot_download from modelscope.utils.logger import get_logger logger = get_logger() @@ -43,6 +45,12 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): def __init__(self, model: Union[Model, str] = None, preprocessor: WavToScp = None, + vad_model: Optional[Union[Model, str]] = None, + vad_model_revision: Optional[str] = None, + punc_model: Optional[Union[Model, str]] = None, + punc_model_revision: Optional[str] = None, + lm_model: Optional[Union[Model, str]] = None, + lm_model_revision: Optional[str] = None, **kwargs): """ Use `model` and `preprocessor` to create an asr pipeline for prediction @@ -55,6 +63,15 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): - A model id in the model hub preprocessor: (list of) Preprocessor object + vad_model (Optional: 'Model' or 'str'): + voice activity detection model from model hub or local + example: 'damo/speech_fsmn_vad_zh-cn-16k-common-pytorch' + punc_model (Optional: 'Model' or 'str'): + punctuation model from model hub or local + example: 'damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch' + lm_model (Optional: 'Model' or 'str'): + language model from model hub or local + example: 'damo/speech_transformer_lm_zh-cn-common-vocab8404-pytorch' output_dir('str'): output dir path batch_size('int'): @@ -85,6 +102,27 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): extra kwargs """ super().__init__(model=model, preprocessor=preprocessor, **kwargs) + self.vad_model = None + self.punc_model = None + self.lm_model = None + if vad_model is not None: + if os.path.exists(vad_model): + self.vad_model = vad_model + else: + self.vad_model = snapshot_download( + vad_model, revision=vad_model_revision) + if punc_model is not None: + if os.path.exists(punc_model): + self.punc_model = punc_model + else: + self.punc_model = snapshot_download( + punc_model, revision=punc_model_revision) + if lm_model is not None: + if os.path.exists(lm_model): + self.lm_model = lm_model + else: + self.lm_model = snapshot_download( + lm_model, revision=lm_model_revision) self.model_cfg = self.model.forward() self.cmd = self.get_cmd(kwargs) @@ -337,6 +375,9 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): cmd['punc_model_file'] = outputs['punc_model_name'] if outputs.__contains__('punc_model_config'): cmd['punc_infer_config'] = outputs['punc_model_config'] + self.load_vad_model(cmd) + self.load_punc_model(cmd) + self.load_lm_model(cmd) user_args_dict = [ 'output_dir', @@ -380,6 +421,53 @@ class AutomaticSpeechRecognitionPipeline(Pipeline): return cmd + def load_vad_model(self, cmd): + if self.vad_model is not None: + logger.info('loading vad model from {0} ...'.format( + self.vad_model)) + config_path = os.path.join(self.vad_model, ModelFile.CONFIGURATION) + model_cfg = json.loads(open(config_path).read()) + model_dir = os.path.dirname(config_path) + cmd['vad_model_file'] = os.path.join( + model_dir, + model_cfg['model']['model_config']['vad_model_name']) + cmd['vad_infer_config'] = os.path.join( + model_dir, + model_cfg['model']['model_config']['vad_model_config']) + cmd['vad_cmvn_file'] = os.path.join( + model_dir, model_cfg['model']['model_config']['vad_mvn_file']) + if 'vad' not in cmd['mode']: + cmd['mode'] = cmd['mode'] + '_vad' + + def load_punc_model(self, cmd): + if self.punc_model is not None: + logger.info('loading punctuation model from {0} ...'.format( + self.punc_model)) + config_path = os.path.join(self.punc_model, + ModelFile.CONFIGURATION) + model_cfg = json.loads(open(config_path).read()) + model_dir = os.path.dirname(config_path) + cmd['punc_model_file'] = os.path.join( + model_dir, model_cfg['model']['punc_model_name']) + cmd['punc_infer_config'] = os.path.join( + model_dir, + model_cfg['model']['punc_model_config']['punc_config']) + if 'punc' not in cmd['mode']: + cmd['mode'] = cmd['mode'] + '_punc' + + def load_lm_model(self, cmd): + if self.lm_model is not None: + logger.info('loading language model from {0} ...'.format( + self.lm_model)) + config_path = os.path.join(self.lm_model, ModelFile.CONFIGURATION) + model_cfg = json.loads(open(config_path).read()) + model_dir = os.path.dirname(config_path) + cmd['lm_file'] = os.path.join( + model_dir, model_cfg['model']['model_config']['lm_model_name']) + cmd['lm_train_config'] = os.path.join( + model_dir, + model_cfg['model']['model_config']['lm_model_config']) + def forward(self, inputs: Dict[str, Any]) -> Dict[str, Any]: """Decoding """ diff --git a/modelscope/pipelines/audio/speaker_verification_pipeline.py b/modelscope/pipelines/audio/speaker_verification_pipeline.py index bfcc47c0..39769114 100644 --- a/modelscope/pipelines/audio/speaker_verification_pipeline.py +++ b/modelscope/pipelines/audio/speaker_verification_pipeline.py @@ -113,7 +113,7 @@ class SpeakerVerificationPipeline(Pipeline): 'dtype': 'float32', 'ngpu': 1, # 0: only CPU, ngpu>=1: gpu number if cuda is available 'seed': 0, - 'num_workers': 1, + 'num_workers': 0, 'log_level': 'ERROR', 'key_file': None, 'sv_model_file': sv_model_path, @@ -134,6 +134,7 @@ class SpeakerVerificationPipeline(Pipeline): 'log_level', 'allow_variable_data_keys', 'streaming', + 'num_workers', 'param_dict', ] diff --git a/modelscope/pipelines/audio/voice_activity_detection_pipeline.py b/modelscope/pipelines/audio/voice_activity_detection_pipeline.py new file mode 100644 index 00000000..f3248af1 --- /dev/null +++ b/modelscope/pipelines/audio/voice_activity_detection_pipeline.py @@ -0,0 +1,225 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +from typing import Any, Dict, List, Sequence, Tuple, Union + +import json +from funasr.utils import asr_utils + +from modelscope.metainfo import Pipelines +from modelscope.models import Model +from modelscope.outputs import OutputKeys +from modelscope.pipelines.base import Pipeline +from modelscope.pipelines.builder import PIPELINES +from modelscope.utils.audio.audio_utils import generate_scp_from_url +from modelscope.utils.constant import Frameworks, ModelFile, Tasks +from modelscope.utils.logger import get_logger + +logger = get_logger() + +__all__ = ['VoiceActivityDetectionPipeline'] + + +@PIPELINES.register_module( + Tasks.voice_activity_detection, module_name=Pipelines.vad_inference) +class VoiceActivityDetectionPipeline(Pipeline): + """Voice Activity Detection Inference Pipeline + use `model` to create a Voice Activity Detection pipeline. + + Args: + model: A model instance, or a model local dir, or a model id in the model hub. + kwargs (dict, `optional`): + Extra kwargs passed into the preprocessor's constructor. + + Example: + >>> from modelscope.pipelines import pipeline + >>> pipeline_vad = pipeline( + >>> task=Tasks.voice_activity_detection, model='damo/speech_fsmn_vad_zh-cn-16k-common-pytorch') + >>> audio_in='https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/vad_example.pcm' + >>> print(pipeline_vad(audio_in)) + + """ + + def __init__(self, model: Union[Model, str] = None, **kwargs): + """use `model` to create an vad pipeline for prediction + """ + super().__init__(model=model, **kwargs) + config_path = os.path.join(model, ModelFile.CONFIGURATION) + self.cmd = self.get_cmd(config_path, kwargs) + + from funasr.bin import vad_inference_launch + self.funasr_infer_modelscope = vad_inference_launch.inference_launch( + mode=self.cmd['mode'], + batch_size=self.cmd['batch_size'], + dtype=self.cmd['dtype'], + ngpu=self.cmd['ngpu'], + seed=self.cmd['seed'], + num_workers=self.cmd['num_workers'], + log_level=self.cmd['log_level'], + key_file=self.cmd['key_file'], + vad_infer_config=self.cmd['vad_infer_config'], + vad_model_file=self.cmd['vad_model_file'], + vad_cmvn_file=self.cmd['vad_cmvn_file']) + + def __call__(self, + audio_in: Union[str, bytes], + audio_fs: int = None, + recog_type: str = None, + audio_format: str = None, + output_dir: str = None, + param_dict: dict = None) -> Dict[str, Any]: + """ + Decoding the input audios + Args: + audio_in('str' or 'bytes'): + - A string containing a local path to a wav file + - A string containing a local path to a scp + - A string containing a wav url + - A bytes input + audio_fs('int'): + frequency of sample + recog_type('str'): + recog type for wav file or datasets file ('wav', 'test', 'dev', 'train') + audio_format('str'): + audio format ('pcm', 'scp', 'kaldi_ark', 'tfrecord') + output_dir('str'): + output dir + param_dict('dict'): + extra kwargs + Return: + A dictionary of result or a list of dictionary of result. + + The dictionary contain the following keys: + - **text** ('str') --The vad result. + """ + self.recog_type = recog_type + self.audio_format = audio_format + self.audio_fs = audio_fs + checking_audio_fs = None + self.raw_inputs = None + if output_dir is not None: + self.cmd['output_dir'] = output_dir + if audio_fs is not None: + self.cmd['fs']['audio_fs'] = audio_fs + if isinstance(audio_in, str): + # for funasr code, generate wav.scp from url or local path + self.audio_in, self.raw_inputs = generate_scp_from_url(audio_in) + elif isinstance(audio_in, bytes): + self.audio_in = audio_in + self.raw_inputs = None + else: + import numpy + import torch + if isinstance(audio_in, torch.Tensor): + self.audio_in = None + self.raw_inputs = audio_in + elif isinstance(audio_in, numpy.ndarray): + self.audio_in = None + self.raw_inputs = audio_in + if output_dir is not None: + self.cmd['output_dir'] = output_dir + if param_dict is not None: + self.cmd['param_dict'] = param_dict + + # set the sample_rate of audio_in if checking_audio_fs is valid + if checking_audio_fs is not None: + self.audio_fs = checking_audio_fs + + if recog_type is None or audio_format is None: + self.recog_type, self.audio_format, self.audio_in = asr_utils.type_checking( + audio_in=self.audio_in, + recog_type=recog_type, + audio_format=audio_format) + + if hasattr(asr_utils, + 'sample_rate_checking') and self.audio_in is not None: + checking_audio_fs = asr_utils.sample_rate_checking( + self.audio_in, self.audio_format) + if checking_audio_fs is not None: + self.audio_fs = checking_audio_fs + output = self.forward(self.audio_in) + result = self.postprocess(output) + return result + + def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + """Postprocessing + """ + rst = {} + for i in range(len(inputs)): + if i == 0: + text = inputs[0]['value'] + if len(text) > 0: + rst[OutputKeys.TEXT] = text + else: + rst[inputs[i]['key']] = inputs[i]['value'] + return rst + + def get_cmd(self, config_path, extra_args) -> Dict[str, Any]: + model_cfg = json.loads(open(config_path).read()) + model_dir = os.path.dirname(config_path) + # generate inference command + vad_model_path = os.path.join( + model_dir, model_cfg['model']['model_config']['vad_model_name']) + vad_model_config = os.path.join( + model_dir, model_cfg['model']['model_config']['vad_model_config']) + vad_cmvn_file = os.path.join( + model_dir, model_cfg['model']['model_config']['vad_mvn_file']) + mode = model_cfg['model']['model_config']['mode'] + cmd = { + 'mode': mode, + 'batch_size': 1, + 'dtype': 'float32', + 'ngpu': 1, # 0: only CPU, ngpu>=1: gpu number if cuda is available + 'seed': 0, + 'num_workers': 0, + 'log_level': 'ERROR', + 'key_file': None, + 'vad_infer_config': vad_model_config, + 'vad_model_file': vad_model_path, + 'vad_cmvn_file': vad_cmvn_file, + 'output_dir': None, + 'param_dict': None, + } + + user_args_dict = [ + 'output_dir', 'batch_size', 'mode', 'ngpu', 'param_dict', + 'num_workers' + ] + + for user_args in user_args_dict: + if user_args in extra_args and extra_args[user_args] is not None: + cmd[user_args] = extra_args[user_args] + + return cmd + + def forward(self, audio_in: Dict[str, Any]) -> Dict[str, Any]: + """Decoding + """ + logger.info('VAD Processing ...') + # generate inputs + data_cmd: Sequence[Tuple[str, str, str]] + if isinstance(self.audio_in, bytes): + data_cmd = [self.audio_in, 'speech', 'bytes'] + elif isinstance(self.audio_in, str): + data_cmd = [self.audio_in, 'speech', 'sound'] + elif self.raw_inputs is not None: + data_cmd = None + self.cmd['name_and_type'] = data_cmd + self.cmd['raw_inputs'] = self.raw_inputs + self.cmd['audio_in'] = self.audio_in + + vad_result = self.run_inference(self.cmd) + + return vad_result + + def run_inference(self, cmd): + vad_result = [] + if self.framework == Frameworks.torch: + vad_result = self.funasr_infer_modelscope( + data_path_and_name_and_type=cmd['name_and_type'], + raw_inputs=cmd['raw_inputs'], + output_dir_v2=cmd['output_dir'], + param_dict=cmd['param_dict']) + else: + raise ValueError('model type is mismatching') + + return vad_result diff --git a/modelscope/utils/constant.py b/modelscope/utils/constant.py index 8c10703c..7a791a5f 100644 --- a/modelscope/utils/constant.py +++ b/modelscope/utils/constant.py @@ -197,6 +197,7 @@ class AudioTasks(object): inverse_text_processing = 'inverse-text-processing' punctuation = 'punctuation' speaker_verification = 'speaker-verification' + voice_activity_detection = 'voice-activity-detection' language_model = 'language-model' diff --git a/tests/msdatasets/test_ms_dataset.py b/tests/msdatasets/test_ms_dataset.py index bbfc6948..2bea0c4c 100644 --- a/tests/msdatasets/test_ms_dataset.py +++ b/tests/msdatasets/test_ms_dataset.py @@ -4,6 +4,7 @@ import unittest from modelscope.models import Model from modelscope.msdatasets import MsDataset +from modelscope.msdatasets.audio.asr_dataset import ASRDataset from modelscope.preprocessors import TextClassificationTransformersPreprocessor from modelscope.preprocessors.base import Preprocessor from modelscope.utils.constant import DEFAULT_DATASET_NAMESPACE, DownloadMode @@ -111,6 +112,12 @@ class MsDatasetTest(unittest.TestCase): drop_remainder=True) print(next(iter(tf_dataset))) + @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') + def test_to_dataset_asr(self): + ms_ds_asr = ASRDataset.load( + 'speech_asr_aishell1_trainsets', namespace='speech_asr') + print(next(iter(ms_ds_asr['train']))) + @unittest.skipUnless(test_level() >= 2, 'skip test in current test level') @require_torch def test_to_torch_dataset_img(self):