asr训练dataset & 单独vad模型推理 & 多模型组合的asr推理

Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/11610526

* add asr dataset

* sv num_workers 0

* add vad pipeline

* add flexible vad/punc/lm model inputs

modelscope/metainfo.py

@@ -416,6 +416,7 @@ class Pipelines(object):
     itn_inference = 'itn-inference'
     punc_inference = 'punc-inference'
     sv_inference = 'sv-inference'
+    vad_inference = 'vad-inference'
     speaker_verification = 'speaker-verification'
     lm_inference = 'language-model'
 

modelscope/models/audio/asr/generic_automatic_speech_recognition.py

@@ -13,6 +13,8 @@ __all__ = ['GenericAutomaticSpeechRecognition']
 
 @MODELS.register_module(
     Tasks.auto_speech_recognition, module_name=Models.generic_asr)
+@MODELS.register_module(
+    Tasks.voice_activity_detection, module_name=Models.generic_asr)
 @MODELS.register_module(Tasks.language_model, module_name=Models.generic_asr)
 class GenericAutomaticSpeechRecognition(Model):
 

modelscope/msdatasets/audio/asr_dataset.py

@@ -0,0 +1,48 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import os
+
+from modelscope.msdatasets.ms_dataset import MsDataset
+
+
+class ASRDataset(MsDataset):
+    """ASR dataset for speech recognition.
+    support load dataset from msdataset hub or local data_dir (including wav.scp and text)
+    For more details, please refer to
+        https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/datasets/ms_dataset.py.
+    """
+
+    @classmethod
+    def load_core(cls, data_dir, data_set):
+        wav_file = os.path.join(data_dir, data_set, 'wav.scp')
+        text_file = os.path.join(data_dir, data_set, 'text')
+        with open(wav_file) as f:
+            wav_lines = f.readlines()
+        with open(text_file) as f:
+            text_lines = f.readlines()
+        data_list = []
+        for wav_line, text_line in zip(wav_lines, text_lines):
+            item = {}
+            item['Audio:FILE'] = wav_line.strip().split()[-1]
+            item['Text:LABEL'] = ' '.join(text_line.strip().split()[1:])
+            data_list.append(item)
+        return data_list
+
+    @classmethod
+    def load(cls,
+             dataset_name,
+             namespace='speech_asr',
+             train_set='train',
+             dev_set='validation'):
+        if os.path.exists(dataset_name):
+            data_dir = dataset_name
+            ds_dict = {}
+            ds_dict['train'] = cls.load_core(data_dir, train_set)
+            ds_dict['validation'] = cls.load_core(data_dir, dev_set)
+            ds_dict['raw_data_dir'] = data_dir
+            return ds_dict
+        else:
+            from modelscope.msdatasets import MsDataset
+            ds_dict = MsDataset.load(
+                dataset_name=dataset_name, namespace=namespace)
+            return ds_dict

modelscope/pipelines/audio/asr_inference_pipeline.py

@@ -1,7 +1,8 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 import os
-from typing import Any, Dict, List, Sequence, Tuple, Union
+from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
 
+import json
 import yaml
 
 from modelscope.metainfo import Pipelines
@@ -13,7 +14,8 @@ from modelscope.preprocessors import WavToScp
 from modelscope.utils.audio.audio_utils import (extract_pcm_from_wav,
                                                 generate_scp_from_url,
                                                 load_bytes_from_url)
-from modelscope.utils.constant import Frameworks, Tasks
+from modelscope.utils.constant import Frameworks, ModelFile, Tasks
+from modelscope.utils.hub import snapshot_download
 from modelscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -43,6 +45,12 @@ class AutomaticSpeechRecognitionPipeline(Pipeline):
     def __init__(self,
                  model: Union[Model, str] = None,
                  preprocessor: WavToScp = None,
+                 vad_model: Optional[Union[Model, str]] = None,
+                 vad_model_revision: Optional[str] = None,
+                 punc_model: Optional[Union[Model, str]] = None,
+                 punc_model_revision: Optional[str] = None,
+                 lm_model: Optional[Union[Model, str]] = None,
+                 lm_model_revision: Optional[str] = None,
                  **kwargs):
         """
         Use `model` and `preprocessor` to create an asr pipeline for prediction
@@ -55,6 +63,15 @@ class AutomaticSpeechRecognitionPipeline(Pipeline):
                 - A model id in the model hub
             preprocessor:
                 (list of) Preprocessor object
+            vad_model (Optional: 'Model' or 'str'):
+                voice activity detection model from model hub or local
+                example: 'damo/speech_fsmn_vad_zh-cn-16k-common-pytorch'
+            punc_model (Optional: 'Model' or 'str'):
+                punctuation model from model hub or local
+                example: 'damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch'
+            lm_model (Optional: 'Model' or 'str'):
+                language model from model hub or local
+                example: 'damo/speech_transformer_lm_zh-cn-common-vocab8404-pytorch'
             output_dir('str'):
                 output dir path
             batch_size('int'):
@@ -85,6 +102,27 @@ class AutomaticSpeechRecognitionPipeline(Pipeline):
                 extra kwargs
         """
         super().__init__(model=model, preprocessor=preprocessor, **kwargs)
+        self.vad_model = None
+        self.punc_model = None
+        self.lm_model = None
+        if vad_model is not None:
+            if os.path.exists(vad_model):
+                self.vad_model = vad_model
+            else:
+                self.vad_model = snapshot_download(
+                    vad_model, revision=vad_model_revision)
+        if punc_model is not None:
+            if os.path.exists(punc_model):
+                self.punc_model = punc_model
+            else:
+                self.punc_model = snapshot_download(
+                    punc_model, revision=punc_model_revision)
+        if lm_model is not None:
+            if os.path.exists(lm_model):
+                self.lm_model = lm_model
+            else:
+                self.lm_model = snapshot_download(
+                    lm_model, revision=lm_model_revision)
         self.model_cfg = self.model.forward()
 
         self.cmd = self.get_cmd(kwargs)
@@ -337,6 +375,9 @@ class AutomaticSpeechRecognitionPipeline(Pipeline):
                 cmd['punc_model_file'] = outputs['punc_model_name']
             if outputs.__contains__('punc_model_config'):
                 cmd['punc_infer_config'] = outputs['punc_model_config']
+            self.load_vad_model(cmd)
+            self.load_punc_model(cmd)
+            self.load_lm_model(cmd)
 
             user_args_dict = [
                 'output_dir',
@@ -380,6 +421,53 @@ class AutomaticSpeechRecognitionPipeline(Pipeline):
 
         return cmd
 
+    def load_vad_model(self, cmd):
+        if self.vad_model is not None:
+            logger.info('loading vad model from {0} ...'.format(
+                self.vad_model))
+            config_path = os.path.join(self.vad_model, ModelFile.CONFIGURATION)
+            model_cfg = json.loads(open(config_path).read())
+            model_dir = os.path.dirname(config_path)
+            cmd['vad_model_file'] = os.path.join(
+                model_dir,
+                model_cfg['model']['model_config']['vad_model_name'])
+            cmd['vad_infer_config'] = os.path.join(
+                model_dir,
+                model_cfg['model']['model_config']['vad_model_config'])
+            cmd['vad_cmvn_file'] = os.path.join(
+                model_dir, model_cfg['model']['model_config']['vad_mvn_file'])
+            if 'vad' not in cmd['mode']:
+                cmd['mode'] = cmd['mode'] + '_vad'
+
+    def load_punc_model(self, cmd):
+        if self.punc_model is not None:
+            logger.info('loading punctuation model from {0} ...'.format(
+                self.punc_model))
+            config_path = os.path.join(self.punc_model,
+                                       ModelFile.CONFIGURATION)
+            model_cfg = json.loads(open(config_path).read())
+            model_dir = os.path.dirname(config_path)
+            cmd['punc_model_file'] = os.path.join(
+                model_dir, model_cfg['model']['punc_model_name'])
+            cmd['punc_infer_config'] = os.path.join(
+                model_dir,
+                model_cfg['model']['punc_model_config']['punc_config'])
+            if 'punc' not in cmd['mode']:
+                cmd['mode'] = cmd['mode'] + '_punc'
+
+    def load_lm_model(self, cmd):
+        if self.lm_model is not None:
+            logger.info('loading language model from {0} ...'.format(
+                self.lm_model))
+            config_path = os.path.join(self.lm_model, ModelFile.CONFIGURATION)
+            model_cfg = json.loads(open(config_path).read())
+            model_dir = os.path.dirname(config_path)
+            cmd['lm_file'] = os.path.join(
+                model_dir, model_cfg['model']['model_config']['lm_model_name'])
+            cmd['lm_train_config'] = os.path.join(
+                model_dir,
+                model_cfg['model']['model_config']['lm_model_config'])
+
     def forward(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
         """Decoding
         """

modelscope/pipelines/audio/speaker_verification_pipeline.py

@@ -113,7 +113,7 @@ class SpeakerVerificationPipeline(Pipeline):
             'dtype': 'float32',
             'ngpu': 1,  # 0: only CPU, ngpu>=1: gpu number if cuda is available
             'seed': 0,
-            'num_workers': 1,
+            'num_workers': 0,
             'log_level': 'ERROR',
             'key_file': None,
             'sv_model_file': sv_model_path,
@@ -134,6 +134,7 @@ class SpeakerVerificationPipeline(Pipeline):
             'log_level',
             'allow_variable_data_keys',
             'streaming',
+            'num_workers',
             'param_dict',
         ]
 

modelscope/pipelines/audio/voice_activity_detection_pipeline.py

@@ -0,0 +1,225 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+from typing import Any, Dict, List, Sequence, Tuple, Union
+
+import json
+from funasr.utils import asr_utils
+
+from modelscope.metainfo import Pipelines
+from modelscope.models import Model
+from modelscope.outputs import OutputKeys
+from modelscope.pipelines.base import Pipeline
+from modelscope.pipelines.builder import PIPELINES
+from modelscope.utils.audio.audio_utils import generate_scp_from_url
+from modelscope.utils.constant import Frameworks, ModelFile, Tasks
+from modelscope.utils.logger import get_logger
+
+logger = get_logger()
+
+__all__ = ['VoiceActivityDetectionPipeline']
+
+
+@PIPELINES.register_module(
+    Tasks.voice_activity_detection, module_name=Pipelines.vad_inference)
+class VoiceActivityDetectionPipeline(Pipeline):
+    """Voice Activity Detection Inference Pipeline
+    use `model` to create a Voice Activity Detection pipeline.
+
+    Args:
+        model: A model instance, or a model local dir, or a model id in the model hub.
+        kwargs (dict, `optional`):
+            Extra kwargs passed into the preprocessor's constructor.
+
+    Example:
+        >>> from modelscope.pipelines import pipeline
+        >>> pipeline_vad = pipeline(
+        >>>    task=Tasks.voice_activity_detection, model='damo/speech_fsmn_vad_zh-cn-16k-common-pytorch')
+        >>> audio_in='https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/vad_example.pcm'
+        >>> print(pipeline_vad(audio_in))
+
+    """
+
+    def __init__(self, model: Union[Model, str] = None, **kwargs):
+        """use `model` to create an vad pipeline for prediction
+        """
+        super().__init__(model=model, **kwargs)
+        config_path = os.path.join(model, ModelFile.CONFIGURATION)
+        self.cmd = self.get_cmd(config_path, kwargs)
+
+        from funasr.bin import vad_inference_launch
+        self.funasr_infer_modelscope = vad_inference_launch.inference_launch(
+            mode=self.cmd['mode'],
+            batch_size=self.cmd['batch_size'],
+            dtype=self.cmd['dtype'],
+            ngpu=self.cmd['ngpu'],
+            seed=self.cmd['seed'],
+            num_workers=self.cmd['num_workers'],
+            log_level=self.cmd['log_level'],
+            key_file=self.cmd['key_file'],
+            vad_infer_config=self.cmd['vad_infer_config'],
+            vad_model_file=self.cmd['vad_model_file'],
+            vad_cmvn_file=self.cmd['vad_cmvn_file'])
+
+    def __call__(self,
+                 audio_in: Union[str, bytes],
+                 audio_fs: int = None,
+                 recog_type: str = None,
+                 audio_format: str = None,
+                 output_dir: str = None,
+                 param_dict: dict = None) -> Dict[str, Any]:
+        """
+        Decoding the input audios
+        Args:
+            audio_in('str' or 'bytes'):
+                - A string containing a local path to a wav file
+                - A string containing a local path to a scp
+                - A string containing a wav url
+                - A bytes input
+            audio_fs('int'):
+                frequency of sample
+            recog_type('str'):
+                recog type for wav file or datasets file ('wav', 'test', 'dev', 'train')
+            audio_format('str'):
+                audio format ('pcm', 'scp', 'kaldi_ark', 'tfrecord')
+            output_dir('str'):
+                output dir
+            param_dict('dict'):
+                extra kwargs
+        Return:
+            A dictionary of result or a list of dictionary of result.
+
+            The dictionary contain the following keys:
+            - **text** ('str') --The vad result.
+        """
+        self.recog_type = recog_type
+        self.audio_format = audio_format
+        self.audio_fs = audio_fs
+        checking_audio_fs = None
+        self.raw_inputs = None
+        if output_dir is not None:
+            self.cmd['output_dir'] = output_dir
+        if audio_fs is not None:
+            self.cmd['fs']['audio_fs'] = audio_fs
+        if isinstance(audio_in, str):
+            # for funasr code, generate wav.scp from url or local path
+            self.audio_in, self.raw_inputs = generate_scp_from_url(audio_in)
+        elif isinstance(audio_in, bytes):
+            self.audio_in = audio_in
+            self.raw_inputs = None
+        else:
+            import numpy
+            import torch
+            if isinstance(audio_in, torch.Tensor):
+                self.audio_in = None
+                self.raw_inputs = audio_in
+            elif isinstance(audio_in, numpy.ndarray):
+                self.audio_in = None
+                self.raw_inputs = audio_in
+        if output_dir is not None:
+            self.cmd['output_dir'] = output_dir
+        if param_dict is not None:
+            self.cmd['param_dict'] = param_dict
+
+        # set the sample_rate of audio_in if checking_audio_fs is valid
+        if checking_audio_fs is not None:
+            self.audio_fs = checking_audio_fs
+
+        if recog_type is None or audio_format is None:
+            self.recog_type, self.audio_format, self.audio_in = asr_utils.type_checking(
+                audio_in=self.audio_in,
+                recog_type=recog_type,
+                audio_format=audio_format)
+
+        if hasattr(asr_utils,
+                   'sample_rate_checking') and self.audio_in is not None:
+            checking_audio_fs = asr_utils.sample_rate_checking(
+                self.audio_in, self.audio_format)
+            if checking_audio_fs is not None:
+                self.audio_fs = checking_audio_fs
+        output = self.forward(self.audio_in)
+        result = self.postprocess(output)
+        return result
+
+    def postprocess(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        """Postprocessing
+        """
+        rst = {}
+        for i in range(len(inputs)):
+            if i == 0:
+                text = inputs[0]['value']
+                if len(text) > 0:
+                    rst[OutputKeys.TEXT] = text
+            else:
+                rst[inputs[i]['key']] = inputs[i]['value']
+        return rst
+
+    def get_cmd(self, config_path, extra_args) -> Dict[str, Any]:
+        model_cfg = json.loads(open(config_path).read())
+        model_dir = os.path.dirname(config_path)
+        # generate inference command
+        vad_model_path = os.path.join(
+            model_dir, model_cfg['model']['model_config']['vad_model_name'])
+        vad_model_config = os.path.join(
+            model_dir, model_cfg['model']['model_config']['vad_model_config'])
+        vad_cmvn_file = os.path.join(
+            model_dir, model_cfg['model']['model_config']['vad_mvn_file'])
+        mode = model_cfg['model']['model_config']['mode']
+        cmd = {
+            'mode': mode,
+            'batch_size': 1,
+            'dtype': 'float32',
+            'ngpu': 1,  # 0: only CPU, ngpu>=1: gpu number if cuda is available
+            'seed': 0,
+            'num_workers': 0,
+            'log_level': 'ERROR',
+            'key_file': None,
+            'vad_infer_config': vad_model_config,
+            'vad_model_file': vad_model_path,
+            'vad_cmvn_file': vad_cmvn_file,
+            'output_dir': None,
+            'param_dict': None,
+        }
+
+        user_args_dict = [
+            'output_dir', 'batch_size', 'mode', 'ngpu', 'param_dict',
+            'num_workers'
+        ]
+
+        for user_args in user_args_dict:
+            if user_args in extra_args and extra_args[user_args] is not None:
+                cmd[user_args] = extra_args[user_args]
+
+        return cmd
+
+    def forward(self, audio_in: Dict[str, Any]) -> Dict[str, Any]:
+        """Decoding
+        """
+        logger.info('VAD Processing ...')
+        # generate inputs
+        data_cmd: Sequence[Tuple[str, str, str]]
+        if isinstance(self.audio_in, bytes):
+            data_cmd = [self.audio_in, 'speech', 'bytes']
+        elif isinstance(self.audio_in, str):
+            data_cmd = [self.audio_in, 'speech', 'sound']
+        elif self.raw_inputs is not None:
+            data_cmd = None
+        self.cmd['name_and_type'] = data_cmd
+        self.cmd['raw_inputs'] = self.raw_inputs
+        self.cmd['audio_in'] = self.audio_in
+
+        vad_result = self.run_inference(self.cmd)
+
+        return vad_result
+
+    def run_inference(self, cmd):
+        vad_result = []
+        if self.framework == Frameworks.torch:
+            vad_result = self.funasr_infer_modelscope(
+                data_path_and_name_and_type=cmd['name_and_type'],
+                raw_inputs=cmd['raw_inputs'],
+                output_dir_v2=cmd['output_dir'],
+                param_dict=cmd['param_dict'])
+        else:
+            raise ValueError('model type is mismatching')
+
+        return vad_result

modelscope/utils/constant.py

@@ -197,6 +197,7 @@ class AudioTasks(object):
     inverse_text_processing = 'inverse-text-processing'
     punctuation = 'punctuation'
     speaker_verification = 'speaker-verification'
+    voice_activity_detection = 'voice-activity-detection'
     language_model = 'language-model'
 
 

tests/msdatasets/test_ms_dataset.py

@@ -4,6 +4,7 @@ import unittest
 
 from modelscope.models import Model
 from modelscope.msdatasets import MsDataset
+from modelscope.msdatasets.audio.asr_dataset import ASRDataset
 from modelscope.preprocessors import TextClassificationTransformersPreprocessor
 from modelscope.preprocessors.base import Preprocessor
 from modelscope.utils.constant import DEFAULT_DATASET_NAMESPACE, DownloadMode
@@ -111,6 +112,12 @@ class MsDatasetTest(unittest.TestCase):
             drop_remainder=True)
         print(next(iter(tf_dataset)))
 
+    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    def test_to_dataset_asr(self):
+        ms_ds_asr = ASRDataset.load(
+            'speech_asr_aishell1_trainsets', namespace='speech_asr')
+        print(next(iter(ms_ds_asr['train'])))
+
     @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
     @require_torch
     def test_to_torch_dataset_img(self):