add audio codec and codec-based TTS model

Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/15128959

modelscope/metainfo.py

@@ -206,6 +206,8 @@ class Models(object):
     cluster_backend = 'cluster-backend'
     rdino_tdnn_sv = 'rdino_ecapa-tdnn-sv'
     generic_lm = 'generic-lm'
+    audio_quantization = 'audio-quantization'
+    laura_codec = 'laura-codec'
 
     # multi-modal models
     ofa = 'ofa'
@@ -545,6 +547,9 @@ class Pipelines(object):
     segmentation_clustering = 'segmentation-clustering'
     lm_inference = 'language-score-prediction'
     speech_timestamp_inference = 'speech-timestamp-inference'
+    audio_quantization = 'audio-quantization'
+    audio_quantization_inference = 'audio-quantization-inference'
+    laura_codec_tts_inference = 'laura-codec-tts-inference'
 
     # multi-modal tasks
     image_captioning = 'image-captioning'

modelscope/models/audio/quantization/__init__.py

@@ -0,0 +1,22 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import TYPE_CHECKING
+
+from modelscope.utils.import_utils import LazyImportModule
+
+if TYPE_CHECKING:
+    from .generic_audio_quantization import GenericAudioQuantization
+
+else:
+    _import_structure = {
+        'generic_audio_quantization': ['GenericAudioQuantization'],
+    }
+
+    import sys
+
+    sys.modules[__name__] = LazyImportModule(
+        __name__,
+        globals()['__file__'],
+        _import_structure,
+        module_spec=__spec__,
+        extra_objects={},
+    )

modelscope/models/audio/quantization/generic_audio_quantization.py

@@ -0,0 +1,45 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import os
+from typing import Any, Dict
+
+from modelscope.metainfo import Models
+from modelscope.models.base import Model
+from modelscope.models.builder import MODELS
+from modelscope.utils.constant import Frameworks, Tasks
+
+__all__ = ['GenericAudioQuantization']
+
+
+@MODELS.register_module(
+    Tasks.audio_quantization, module_name=Models.audio_quantization)
+class GenericAudioQuantization(Model):
+
+    def __init__(self, model_dir: str, model_name: str,
+                 model_config: Dict[str, Any], *args, **kwargs):
+        """initialize the info of model.
+
+        Args:
+            model_dir (str): the model path.
+            model_name (str): the itn model name from configuration.json
+            model_config (Dict[str, Any]): the detail config about model from configuration.json
+        """
+        super().__init__(model_dir, model_name, model_config, *args, **kwargs)
+        self.model_cfg = {
+            # the recognition model dir path
+            'model_workspace': model_dir,
+            # the itn model name
+            'model_name': model_name,
+            # the am model file path
+            'model_path': os.path.join(model_dir, model_name),
+            # the recognition model config dict
+            'model_config': model_config
+        }
+
+    def forward(self) -> Dict[str, Any]:
+        """
+          just return the model config
+
+        """
+
+        return self.model_cfg

modelscope/models/audio/tts/__init__.py

@@ -5,9 +5,13 @@ from modelscope.utils.import_utils import LazyImportModule
 
 if TYPE_CHECKING:
     from .sambert_hifi import SambertHifigan
+    from .laura_codec import LauraCodecGenModel
 
 else:
-    _import_structure = {'sambert_hifi': ['SambertHifigan']}
+    _import_structure = {
+        'sambert_hifi': ['SambertHifigan'],
+        'laura_codec': ['LauraCodecGenModel'],
+    }
     import sys
     sys.modules[__name__] = LazyImportModule(
         __name__,

modelscope/models/audio/tts/laura_codec.py

@@ -0,0 +1,44 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import os
+from typing import Any, Dict
+
+from modelscope.metainfo import Models
+from modelscope.models.base import Model
+from modelscope.models.builder import MODELS
+from modelscope.utils.constant import Frameworks, Tasks
+
+__all__ = ['LauraCodecGenModel']
+
+
+@MODELS.register_module(Tasks.text_to_speech, module_name=Models.laura_codec)
+class LauraCodecGenModel(Model):
+
+    def __init__(self, model_dir: str, model_name: str,
+                 model_config: Dict[str, Any], *args, **kwargs):
+        """initialize the info of model.
+
+        Args:
+            model_dir (str): the model path.
+            model_name (str): the itn model name from configuration.json
+            model_config (Dict[str, Any]): the detail config about model from configuration.json
+        """
+        super().__init__(model_dir, model_name, model_config, *args, **kwargs)
+        self.model_cfg = {
+            # the recognition model dir path
+            'model_workspace': model_dir,
+            # the itn model name
+            'model_name': model_name,
+            # the am model file path
+            'model_path': os.path.join(model_dir, model_name),
+            # the recognition model config dict
+            'model_config': model_config
+        }
+
+    def forward(self) -> Dict[str, Any]:
+        """
+          just return the model config
+
+        """
+
+        return self.model_cfg

modelscope/pipelines/audio/audio_quantization_pipeline.py

@@ -0,0 +1,229 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+import shutil
+from typing import Any, Dict, List, Sequence, Tuple, Union
+
+import numpy as np
+import yaml
+
+from modelscope.metainfo import Pipelines
+from modelscope.models import Model
+from modelscope.outputs import OutputKeys
+from modelscope.pipelines.base import Pipeline
+from modelscope.pipelines.builder import PIPELINES
+from modelscope.utils.audio.audio_utils import (generate_scp_from_url,
+                                                update_local_model)
+from modelscope.utils.constant import Frameworks, Tasks
+from modelscope.utils.logger import get_logger
+
+logger = get_logger()
+
+__all__ = ['AudioQuantizationPipeline']
+
+
+@PIPELINES.register_module(
+    Tasks.audio_quantization,
+    module_name=Pipelines.audio_quantization_inference)
+class AudioQuantizationPipeline(Pipeline):
+    """Audio Quantization Inference Pipeline
+    use `model` to create a audio quantization pipeline.
+
+    Args:
+        model (AudioQuantizationPipeline): A model instance, or a model local dir, or a model id in the model hub.
+        kwargs (dict, `optional`):
+            Extra kwargs passed into the preprocessor's constructor.
+    Examples:
+        >>> from modelscope.pipelines import pipeline
+        >>> from modelscope.utils.constant import Tasks
+        >>> pipeline_aq = pipeline(
+        >>>    task=Tasks.audio_quantization,
+        >>>    model='damo/audio_codec-encodec-zh_en-general-16k-nq32ds640-pytorch'
+        >>> )
+        >>> audio_in='example.wav'
+        >>> print(pipeline_aq(audio_in))
+
+    """
+
+    def __init__(self,
+                 model: Union[Model, str] = None,
+                 ngpu: int = 1,
+                 **kwargs):
+        """use `model` to create an asr pipeline for prediction
+        """
+        super().__init__(model=model, **kwargs)
+        self.model_cfg = self.model.forward()
+        self.cmd = self.get_cmd(kwargs, model)
+
+        from funcodec.bin import codec_inference
+        self.funasr_infer_modelscope = codec_inference.inference_modelscope(
+            mode=self.cmd['mode'],
+            output_dir=self.cmd['output_dir'],
+            batch_size=self.cmd['batch_size'],
+            dtype=self.cmd['dtype'],
+            ngpu=ngpu,
+            seed=self.cmd['seed'],
+            num_workers=self.cmd['num_workers'],
+            log_level=self.cmd['log_level'],
+            key_file=self.cmd['key_file'],
+            config_file=self.cmd['config_file'],
+            model_file=self.cmd['model_file'],
+            model_tag=self.cmd['model_tag'],
+            allow_variable_data_keys=self.cmd['allow_variable_data_keys'],
+            streaming=self.cmd['streaming'],
+            sampling_rate=self.cmd['sampling_rate'],
+            bit_width=self.cmd['bit_width'],
+            use_scale=self.cmd['use_scale'],
+            param_dict=self.cmd['param_dict'],
+            **kwargs,
+        )
+
+    def __call__(self,
+                 audio_in: Union[tuple, str, Any] = None,
+                 output_dir: str = None,
+                 param_dict: dict = None) -> Dict[str, Any]:
+        if len(audio_in) == 0:
+            raise ValueError('The input should not be null.')
+        else:
+            self.audio_in = audio_in
+        if output_dir is not None:
+            self.cmd['output_dir'] = output_dir
+        self.cmd['param_dict'] = param_dict
+
+        output = self.forward(self.audio_in)
+        result = self.postprocess(output)
+        return result
+
+    def postprocess(self, inputs: list) -> Dict[str, Any]:
+        """Postprocessing
+        """
+        rst = {}
+        for i in range(len(inputs)):
+            if len(inputs) == 1 and i == 0:
+                recon_wav = inputs[0]['value']
+                output_wav = recon_wav.cpu().numpy()[0]
+                output_wav = (output_wav * (2**15)).astype(np.int16)
+                rst[OutputKeys.OUTPUT_WAV] = output_wav
+            else:
+                # for multiple inputs
+                rst[inputs[i]['key']] = inputs[i]['value']
+        return rst
+
+    def get_cmd(self, extra_args, model_path) -> Dict[str, Any]:
+        # generate asr inference command
+        mode = self.model_cfg['model_config']['mode']
+        _model_path = os.path.join(
+            self.model_cfg['model_workspace'],
+            self.model_cfg['model_config']['model_file'])
+        _model_config = os.path.join(
+            self.model_cfg['model_workspace'],
+            self.model_cfg['model_config']['config_file'])
+        update_local_model(self.model_cfg['model_config'], model_path,
+                           extra_args)
+        cmd = {
+            'mode': mode,
+            'output_dir': None,
+            'batch_size': 1,
+            'dtype': 'float32',
+            'ngpu': 1,  # 0: only CPU, ngpu>=1: gpu number if cuda is available
+            'seed': 0,
+            'num_workers': 0,
+            'log_level': 'ERROR',
+            'key_file': None,
+            'model_file': _model_path,
+            'config_file': _model_config,
+            'model_tag': None,
+            'allow_variable_data_keys': True,
+            'streaming': False,
+            'sampling_rate': 16000,
+            'bit_width': 8000,
+            'use_scale': True,
+            'param_dict': None,
+        }
+        user_args_dict = [
+            'output_dir',
+            'batch_size',
+            'ngpu',
+            'log_level',
+            'allow_variable_data_keys',
+            'streaming',
+            'num_workers',
+            'sampling_rate',
+            'bit_width',
+            'use_scale',
+            'param_dict',
+        ]
+
+        # re-write the config with configure.json
+        for user_args in user_args_dict:
+            if (user_args in self.model_cfg['model_config']
+                    and self.model_cfg['model_config'][user_args] is not None):
+                if isinstance(cmd[user_args], dict) and isinstance(
+                        self.model_cfg['model_config'][user_args], dict):
+                    cmd[user_args].update(
+                        self.model_cfg['model_config'][user_args])
+                else:
+                    cmd[user_args] = self.model_cfg['model_config'][user_args]
+
+        # rewrite the config with user args
+        for user_args in user_args_dict:
+            if user_args in extra_args:
+                if extra_args.get(user_args) is not None:
+                    if isinstance(cmd[user_args], dict) and isinstance(
+                            extra_args[user_args], dict):
+                        cmd[user_args].update(extra_args[user_args])
+                    else:
+                        cmd[user_args] = extra_args[user_args]
+                del extra_args[user_args]
+
+        return cmd
+
+    def forward(self, audio_in: Union[tuple, str, Any] = None) -> list:
+        """Decoding
+        """
+        # log  file_path/url or tuple (str, str)
+        if isinstance(audio_in, str):
+            logger.info(f'Audio Quantization Processing: {audio_in} ...')
+        else:
+            logger.info(
+                f'Audio Quantization Processing: {str(audio_in)[:100]} ...')
+
+        data_cmd, raw_inputs = None, None
+        if isinstance(audio_in, str):
+            # for scp inputs
+            if len(audio_in.split(',')) == 3:
+                data_cmd = [tuple(audio_in.split(','))]
+            # for single-file inputs
+            else:
+                audio_scp, _ = generate_scp_from_url(audio_in)
+                raw_inputs = audio_scp
+        # for raw bytes
+        elif isinstance(audio_in, bytes):
+            data_cmd = (audio_in, 'speech', 'bytes')
+        # for ndarray and tensor inputs
+        else:
+            import torch
+            import numpy as np
+            if isinstance(audio_in, torch.Tensor):
+                raw_inputs = audio_in
+            elif isinstance(audio_in, np.ndarray):
+                raw_inputs = audio_in
+            else:
+                raise TypeError('Unsupported data type.')
+
+        self.cmd['name_and_type'] = data_cmd
+        self.cmd['raw_inputs'] = raw_inputs
+        result = self.run_inference(self.cmd)
+
+        return result
+
+    def run_inference(self, cmd):
+        if self.framework == Frameworks.torch:
+            sv_result = self.funasr_infer_modelscope(
+                data_path_and_name_and_type=cmd['name_and_type'],
+                raw_inputs=cmd['raw_inputs'],
+                output_dir_v2=cmd['output_dir'],
+                param_dict=cmd['param_dict'])
+        else:
+            raise ValueError('model type is mismatching')
+
+        return sv_result

modelscope/pipelines/audio/codec_based_synthesis_pipeline.py

@@ -0,0 +1,276 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import os
+from typing import Any, Dict, Optional, Union
+
+import json
+import numpy as np
+
+from modelscope.metainfo import Pipelines
+from modelscope.models import Model
+from modelscope.outputs import OutputKeys
+from modelscope.pipelines.base import Pipeline
+from modelscope.pipelines.builder import PIPELINES
+from modelscope.utils.audio.audio_utils import (generate_scp_from_url,
+                                                update_local_model)
+from modelscope.utils.constant import Frameworks, ModelFile, Tasks
+from modelscope.utils.hub import snapshot_download
+from modelscope.utils.logger import get_logger
+
+__all__ = ['LauraCodecTTSPipeline']
+
+logger = get_logger()
+
+
+@PIPELINES.register_module(
+    Tasks.text_to_speech, module_name=Pipelines.laura_codec_tts_inference)
+class LauraCodecTTSPipeline(Pipeline):
+    """Laura-style Codec-based TTS Inference Pipeline
+    use `model` to create a TTS pipeline.
+
+    Args:
+        model (LauraCodecTTSPipeline): A model instance, or a model local dir, or a model id in the model hub.
+        kwargs (dict, `optional`):
+            Extra kwargs passed into the preprocessor's constructor.
+    Examples:
+        >>> from modelscope.pipelines import pipeline
+        >>> from modelscope.utils.constant import Tasks
+        >>> my_pipeline = pipeline(
+        >>>    task=Tasks.text_to_speech,
+        >>>    model='damo/speech_synthesizer-laura-en-libritts-16k-codec_nq2-pytorch'
+        >>> )
+        >>> text='nothing was to be done but to put about, and return in disappointment towards the north.'
+        >>> prompt_text='one of these is context'
+        >>> prompt_speech='example/prompt.wav'
+        >>> print(my_pipeline(text))
+
+    """
+
+    def __init__(self,
+                 model: Union[Model, str] = None,
+                 codec_model: Optional[Union[Model, str]] = None,
+                 codec_model_revision: Optional[str] = None,
+                 ngpu: int = 1,
+                 **kwargs):
+        """use `model` to create an asr pipeline for prediction
+        """
+        super().__init__(model=model, **kwargs)
+        self.model_cfg = self.model.forward()
+        self.codec_model = codec_model
+        self.codec_model_revision = codec_model_revision
+        self.cmd = self.get_cmd(kwargs, model)
+
+        from funcodec.bin import text2audio_inference
+        self.funasr_infer_modelscope = text2audio_inference.inference_func(
+            mode=self.cmd['mode'],
+            output_dir=self.cmd['output_dir'],
+            batch_size=self.cmd['batch_size'],
+            dtype=self.cmd['dtype'],
+            ngpu=ngpu,
+            seed=self.cmd['seed'],
+            num_workers=self.cmd['num_workers'],
+            log_level=self.cmd['log_level'],
+            key_file=self.cmd['key_file'],
+            config_file=self.cmd['config_file'],
+            model_file=self.cmd['model_file'],
+            model_tag=self.cmd['model_tag'],
+            allow_variable_data_keys=self.cmd['allow_variable_data_keys'],
+            streaming=self.cmd['streaming'],
+            text_emb_model=self.cmd['text_emb_model'],
+            beam_size=self.cmd['beam_size'],
+            sampling=self.cmd['sampling'],
+            continual=self.cmd['continual'],
+            tokenize_to_phone=self.cmd['tokenize_to_phone'],
+            exclude_prompt=self.cmd['exclude_prompt'],
+            codec_config_file=self.cmd['codec_config_file'],
+            codec_model_file=self.cmd['codec_model_file'],
+            param_dict=self.cmd['param_dict'])
+
+    def __call__(self,
+                 text: Union[tuple, str, Any] = None,
+                 prompt_text: Union[tuple, str, Any] = None,
+                 prompt_audio: Union[tuple, str, Any] = None,
+                 output_dir: str = None,
+                 param_dict: dict = None) -> Dict[str, Any]:
+        if len(text) == 0:
+            raise ValueError('The input should not be null.')
+        if output_dir is not None:
+            self.cmd['output_dir'] = output_dir
+        self.cmd['param_dict'] = param_dict
+
+        output = self.forward(text, prompt_text, prompt_audio)
+        result = self.postprocess(output)
+        return result
+
+    def postprocess(self, inputs: list) -> Dict[str, Any]:
+        """Postprocessing
+        """
+        rst = {}
+        for i in range(len(inputs)):
+            if len(inputs) == 1 and i == 0:
+                recon_wav = inputs[0]['value']['gen']
+                rst[OutputKeys.OUTPUT_WAV] = recon_wav.cpu().numpy()[0]
+            else:
+                # for multiple inputs
+                rst[inputs[i]['key']] = inputs[i]['value']['gen']
+        return rst
+
+    def load_codec_model(self, cmd):
+        if self.codec_model is not None and self.codec_model != '':
+            if os.path.exists(self.codec_model):
+                codec_model = self.codec_model
+            else:
+                codec_model = snapshot_download(
+                    self.codec_model, revision=self.codec_model_revision)
+            logger.info('loading codec model from {0} ...'.format(codec_model))
+            config_path = os.path.join(codec_model, ModelFile.CONFIGURATION)
+            model_cfg = json.loads(open(config_path).read())
+            model_dir = os.path.dirname(config_path)
+            cmd['codec_model_file'] = os.path.join(
+                model_dir, model_cfg['model']['model_config']['model_file'])
+            cmd['codec_config_file'] = os.path.join(
+                model_dir, model_cfg['model']['model_config']['config_file'])
+
+    def get_cmd(self, extra_args, model_path) -> Dict[str, Any]:
+        # generate asr inference command
+        mode = self.model_cfg['model_config']['mode']
+        _model_path = os.path.join(
+            self.model_cfg['model_workspace'],
+            self.model_cfg['model_config']['model_file'])
+        _model_config = os.path.join(
+            self.model_cfg['model_workspace'],
+            self.model_cfg['model_config']['config_file'])
+        update_local_model(self.model_cfg['model_config'], model_path,
+                           extra_args)
+
+        cmd = {
+            'mode': mode,
+            'output_dir': None,
+            'batch_size': 1,
+            'dtype': 'float32',
+            'ngpu': 1,  # 0: only CPU, ngpu>=1: gpu number if cuda is available
+            'seed': 0,
+            'num_workers': 0,
+            'log_level': 'ERROR',
+            'key_file': None,
+            'model_file': _model_path,
+            'config_file': _model_config,
+            'model_tag': None,
+            'allow_variable_data_keys': True,
+            'streaming': False,
+            'beam_size': 1,
+            'sampling': 25,
+            'text_emb_model': None,
+            'continual': True,
+            'tokenize_to_phone': True,
+            'exclude_prompt': True,
+            'codec_model_file': None,
+            'codec_config_file': None,
+            'param_dict': None,
+        }
+        user_args_dict = [
+            'output_dir',
+            'batch_size',
+            'ngpu',
+            'log_level',
+            'allow_variable_data_keys',
+            'streaming',
+            'num_workers',
+            'sampling_rate',
+            'bit_width',
+            'use_scale',
+            'param_dict',
+        ]
+
+        model_config = self.model_cfg['model_config']
+        if model_config.__contains__(
+                'codec_model') and self.codec_model is None:
+            self.codec_model = model_config['codec_model']
+        if model_config.__contains__(
+                'codec_model_revision') and self.codec_model_revision is None:
+            self.codec_model_revision = model_config['codec_model_revision']
+        self.load_codec_model(cmd)
+
+        # re-write the config with configure.json
+        for user_args in user_args_dict:
+            if (user_args in self.model_cfg['model_config']
+                    and self.model_cfg['model_config'][user_args] is not None):
+                if isinstance(cmd[user_args], dict) and isinstance(
+                        self.model_cfg['model_config'][user_args], dict):
+                    cmd[user_args].update(
+                        self.model_cfg['model_config'][user_args])
+                else:
+                    cmd[user_args] = self.model_cfg['model_config'][user_args]
+
+        # rewrite the config with user args
+        for user_args in user_args_dict:
+            if user_args in extra_args:
+                if extra_args.get(user_args) is not None:
+                    if isinstance(cmd[user_args], dict) and isinstance(
+                            extra_args[user_args], dict):
+                        cmd[user_args].update(extra_args[user_args])
+                    else:
+                        cmd[user_args] = extra_args[user_args]
+                del extra_args[user_args]
+
+        return cmd
+
+    def forward(self,
+                text: Union[tuple, str, Any] = None,
+                prompt_text: Union[tuple, str, Any] = None,
+                prompt_audio: Union[tuple, str, Any] = None,
+                **forward_params) -> list:
+        """Decoding
+        """
+        if isinstance(text, str):
+            logger.info(f'Generate speech for: {text} ...')
+
+        data_cmd, raw_inputs = None, None
+        # process text input
+        # for scp inputs
+        if len(text.split(',')) == 3:
+            data_cmd = [tuple(text.split(','))]
+        # for single-file inputs
+        else:
+            raw_inputs = [text]
+
+        if prompt_text is not None and prompt_audio is not None:
+            if len(prompt_text.split(',')) == 3:
+                data_cmd.append(tuple(prompt_text.split(',')))
+            else:
+                raw_inputs.append(prompt_text)
+
+            if isinstance(prompt_audio, str):
+                if len(prompt_audio.split(',')) == 3:
+                    data_cmd.append(tuple(prompt_audio.split(',')))
+                else:
+                    audio_path, _ = generate_scp_from_url(prompt_audio)
+                    raw_inputs.append(audio_path)
+            # for ndarray and tensor inputs
+            else:
+                import torch
+                if isinstance(prompt_audio, torch.Tensor):
+                    raw_inputs.append(prompt_audio.numpy())
+                elif isinstance(prompt_audio, np.ndarray):
+                    raw_inputs.append(prompt_audio)
+                else:
+                    raise TypeError(
+                        f'Unsupported prompt audio type {type(prompt_audio)}.')
+
+        self.cmd['name_and_type'] = data_cmd
+        self.cmd['raw_inputs'] = raw_inputs
+        result = self.run_inference(self.cmd)
+
+        return result
+
+    def run_inference(self, cmd):
+        if self.framework == Frameworks.torch:
+            sv_result = self.funasr_infer_modelscope(
+                data_path_and_name_and_type=cmd['name_and_type'],
+                raw_inputs=cmd['raw_inputs'],
+                output_dir_v2=cmd['output_dir'],
+                param_dict=cmd['param_dict'])
+        else:
+            raise ValueError('model type is mismatching')
+
+        return sv_result

modelscope/pipelines/audio/text_to_speech_pipeline.py

@@ -1,16 +1,15 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from typing import Any, Dict, List
+from typing import Any, Dict
 
 import numpy as np
 
 from modelscope.metainfo import Pipelines
-from modelscope.models import Model
 from modelscope.models.audio.tts import SambertHifigan
 from modelscope.outputs import OutputKeys
 from modelscope.pipelines.base import Input, InputModel, Pipeline
 from modelscope.pipelines.builder import PIPELINES
-from modelscope.utils.constant import Fields, Tasks
+from modelscope.utils.constant import Tasks
 
 __all__ = ['TextToSpeechSambertHifiganPipeline']
 

modelscope/utils/constant.py

@@ -242,6 +242,7 @@ class AudioTasks(object):
     speaker_verification = 'speaker-verification'
     speech_language_recognition = 'speech-language-recognition'
     speaker_diarization = 'speaker-diarization'
+    audio_quantization = 'audio-quantization'
     voice_activity_detection = 'voice-activity-detection'
     language_score_prediction = 'language-score-prediction'
     speech_timestamp = 'speech-timestamp'

modelscope/utils/pipeline_schema.json

@@ -137,6 +137,27 @@
             }
         }
     },
+    "audio-quantization": {
+        "input": {
+            "type": "object",
+            "properties": {
+                "wav": {
+                    "type": "string",
+                    "description": "Base64 encoded audio file or url string.."
+                }
+            }
+        },
+        "parameters": {},
+        "output": {
+            "type": "object",
+            "properties": {
+                "output_wav": {
+                    "type": "string",
+                    "description": "The base64 encoded WAV."
+                }
+            }
+        }
+    },
     "bad-image-detecting": {
         "input": {
             "type": "object",

requirements/audio.txt

@@ -2,3 +2,4 @@
 -r audio/audio_kws.txt
 -r audio/audio_signal.txt
 -r audio/audio_tts.txt
+-r audio/audio_codec.txt

requirements/audio/audio_codec.txt

@@ -0,0 +1 @@
+funcodec>=0.2.0