diff --git a/configs/config.py b/configs/config.py index a60868f..525fec9 100644 --- a/configs/config.py +++ b/configs/config.py @@ -10,6 +10,8 @@ from tools.file_io import read_text import torch import logging +from tools.cuda_graph import configure_cuda_graph + logger = logging.getLogger(__name__) @@ -126,6 +128,12 @@ if infer_device.type != "cuda": ) +# Run a real capture/replay probe on the selected inference device. Both +# application entry points import this module, so downstream inference code +# receives one consistent 0/1 switch without duplicating device checks. +CUDA_GRAPH_AVAILABLE = configure_cuda_graph(infer_device) + + CONFIGS_DIR = Path(__file__).resolve().parent MODEL_CONFIG_FILES = ( "v1/32k.json", @@ -152,6 +160,7 @@ class Config: self.device = str(infer_device) self.dtype = infer_dtype self.is_half = infer_dtype == torch.float16 + self.cuda_graph = CUDA_GRAPH_AVAILABLE self.n_cpu = 0 self.gpu_name = None self.json_config = self.load_config_json() diff --git a/i18n/locale/en_US.json b/i18n/locale/en_US.json index ac2a73c..2fbd0d6 100644 --- a/i18n/locale/en_US.json +++ b/i18n/locale/en_US.json @@ -1,5 +1,5 @@ -{ - "人声提取激进程度": "Vocal extraction aggressiveness", +{ + "人声提取激进程度": "Vocal extraction aggressiveness", "%s → 成功": "%s → Success", "%s运行中，请先停止该任务": "%s is running; stop it before starting another task", "%s进程已终止": "The %s process has been terminated", @@ -49,8 +49,8 @@ "[索引训练][失败] 聚类失败，将使用原始特征继续\n%s": "[Index training][Failed] Clustering failed; continuing with the original features\n%s", "[索引训练][失败] 请先进行特征提取": "[Index training][Failed] Extract features first", "ckpt处理": "ckpt Processing", - "index文件路径不可包含中文": "The index file path cannot contain Chinese characters", - "pth文件路径不可包含中文": "The .pth file path cannot contain Chinese characters", + "index文件路径不可包含中文": "The index file path cannot contain Chinese characters", + "pth文件路径不可包含中文": "The .pth file path cannot contain Chinese characters", "rmvpe卡号配置：以-分隔输入使用的不同进程卡号,例如0-0-1使用在卡0上跑2个进程并在卡1上跑1个进程": "Enter the GPU index(es) separated by '-', e.g., 0-0-1 to use 2 processes in GPU0 and 1 process in GPU1", "step1: 填写实验配置. 实验数据放在logs下, 每个实验一个文件夹, 需手工输入实验名路径, 内含实验配置, 日志, 训练得到的模型文件. ": "Step 1: Fill in the experimental configuration. Experimental data is stored in the 'logs' folder, with each experiment having a separate folder. Manually enter the experiment name path, which contains the experimental configuration, logs, and trained model files.", "step1:正在处理数据": "Step 1: Processing data", @@ -71,8 +71,8 @@ "任务": "Task", "伴奏人声分离&去混响&去回声": "Vocals/Accompaniment Separation & Reverberation Removal", "使用显卡：%s": "GPUs in use: %s", - "使用模型采样率": "Use model sample rate", - "使用设备采样率": "Use device sample rate", + "使用模型采样率": "Use model sample rate", + "使用设备采样率": "Use device sample rate", "保存名": "Save name:", "保存的文件名, 默认空为和源文件同名": "Save file name (default: same as the source file):", "保存的模型名不带后缀": "Saved model name (without extension):", @@ -177,7 +177,7 @@ "特征": "Features", "特征提取": "Feature extraction", "状态": "Status", - "独占 WASAPI 设备": "Exclusive WASAPI device", + "独占 WASAPI 设备": "Exclusive WASAPI device", "生成器预训练模型不存在，将不使用：assets/pretrained%s/%sG%s.pth": "Generator pretrained model not found; it will not be used: assets/pretrained%s/%sG%s.pth", "男转女推荐+12key, 女转男推荐-12key, 如果音域爆炸导致音色失真也可以自己调整到合适音域. ": "Recommended +12 key for male to female conversion, and -12 key for female to male conversion. If the sound range goes too far and the voice is distorted, you can also adjust it to the appropriate range by yourself.", "目标采样率": "Target sample rate:", @@ -202,7 +202,7 @@ "训练结束, 您可查看控制台训练日志或实验文件夹下的train.log": "Training complete. You can check the training logs in the console or the 'train.log' file under the experiment folder.", "训练设备规则选择的精度：%s": "Training precision selected by device rules: %s", "训练轮次：{} [{:.0f}%]": "Training epoch: {} [{:.0f}%]", - "设备类型": "Device type", + "设备类型": "Device type", "请上传音频文件": "Upload an audio file", "请填写输出文件夹路径": "Enter the output folder path", "请指定说话人id": "Please specify the speaker/singer ID:", @@ -233,7 +233,7 @@ "选择模型": "Select model", "选择索引": "Select index", "选择音高提取算法": "Select the pitch extraction algorithm", - "采样率:": "Sample rate:", + "采样率:": "Sample rate:", "采样长度": "Sample length", "重载设备列表": "Reload device list", "错误信息：%s": "Error details: %s", @@ -257,5 +257,7 @@ "[索引训练] 外部索引链接已存在：%s": "[Index training] External index link already exists: %s", "[索引训练][跳过] added索引已存在：%s": "[Index training][Skipped] added index already exists: %s", "[索引训练][跳过] trained索引已存在：%s": "[Index training][Skipped] trained index already exists: %s", - "当前设备：%s | 推理精度：%s": "Current device: %s | Inference precision: %s" + "当前设备：%s | 推理精度：%s": "Current device: %s | Inference precision: %s", + "正在预热CUDA Graph": "Warming up CUDA Graph", + "CUDA Graph预热完成": "CUDA Graph warm-up complete" } diff --git a/i18n/locale/es_ES.json b/i18n/locale/es_ES.json index 57385cd..18f4f4d 100644 --- a/i18n/locale/es_ES.json +++ b/i18n/locale/es_ES.json @@ -1,5 +1,5 @@ -{ - "人声提取激进程度": "Intensidad de extracción vocal", +{ + "人声提取激进程度": "Intensidad de extracción vocal", "%s → 成功": "%s → Éxito", "%s运行中，请先停止该任务": "%s está en ejecución; deténgalo antes de iniciar otra tarea", "%s进程已终止": "El proceso %s ha finalizado", @@ -257,5 +257,7 @@ "[索引训练] 外部索引链接已存在：%s": "[Entrenamiento de índice] El enlace externo ya existe: %s", "[索引训练][跳过] added索引已存在：%s": "[Entrenamiento de índice][Omitido] El índice added ya existe: %s", "[索引训练][跳过] trained索引已存在：%s": "[Entrenamiento de índice][Omitido] El índice trained ya existe: %s", - "当前设备：%s | 推理精度：%s": "Dispositivo actual: %s | Precisión de inferencia: %s" + "当前设备：%s | 推理精度：%s": "Dispositivo actual: %s | Precisión de inferencia: %s", + "正在预热CUDA Graph": "Preparando CUDA Graph", + "CUDA Graph预热完成": "Preparación de CUDA Graph completada" } diff --git a/i18n/locale/fr_FR.json b/i18n/locale/fr_FR.json index ce1e903..23f6a14 100644 --- a/i18n/locale/fr_FR.json +++ b/i18n/locale/fr_FR.json @@ -1,5 +1,5 @@ -{ - "人声提取激进程度": "Intensité de l’extraction vocale", +{ + "人声提取激进程度": "Intensité de l’extraction vocale", "%s → 成功": "%s → Réussite", "%s运行中，请先停止该任务": "%s est en cours ; arrêtez cette tâche avant d'en démarrer une autre", "%s进程已终止": "Le processus %s a été arrêté", @@ -257,5 +257,7 @@ "[索引训练] 外部索引链接已存在：%s": "[Entraînement de l’index] Le lien externe existe déjà : %s", "[索引训练][跳过] added索引已存在：%s": "[Entraînement de l’index][Ignoré] L’index added existe déjà : %s", "[索引训练][跳过] trained索引已存在：%s": "[Entraînement de l’index][Ignoré] L’index trained existe déjà : %s", - "当前设备：%s | 推理精度：%s": "Périphérique actuel : %s | Précision d’inférence : %s" + "当前设备：%s | 推理精度：%s": "Périphérique actuel : %s | Précision d’inférence : %s", + "正在预热CUDA Graph": "Préchauffage de CUDA Graph", + "CUDA Graph预热完成": "Préchauffage de CUDA Graph terminé" } diff --git a/i18n/locale/it_IT.json b/i18n/locale/it_IT.json index 97f32d0..2cf8ba8 100644 --- a/i18n/locale/it_IT.json +++ b/i18n/locale/it_IT.json @@ -1,5 +1,5 @@ -{ - "人声提取激进程度": "Intensità dell'estrazione vocale", +{ + "人声提取激进程度": "Intensità dell'estrazione vocale", "%s → 成功": "%s → Successo", "%s运行中，请先停止该任务": "%s è in esecuzione; interromperlo prima di avviare un'altra attività", "%s进程已终止": "Il processo %s è stato terminato", @@ -54,7 +54,7 @@ "rmvpe卡号配置：以-分隔输入使用的不同进程卡号,例如0-0-1使用在卡0上跑2个进程并在卡1上跑1个进程": "Configurazione GPU RMVPE separata da trattini; ad esempio 0-0-1 avvia due processi sulla GPU 0 e uno sulla GPU 1", "step1: 填写实验配置. 实验数据放在logs下, 每个实验一个文件夹, 需手工输入实验名路径, 内含实验配置, 日志, 训练得到的模型文件. ": "Passaggio 1: compilare la configurazione sperimentale. ", "step1:正在处理数据": "Passaggio 1: elaborazione dei dati", - "step2:正在提取音高&正在提取特征": "step2:Estrazione dell'intonazione e delle caratteristiche", + "step2:正在提取音高&正在提取特征": "step2:Estrazione dell'intonazione e delle caratteristiche", "step2a: 自动遍历训练文件夹下所有可解码成音频的文件并进行切片归一化, 在实验目录下生成2个wav文件夹; 暂时只支持单人训练. ": "Passaggio 2a: attraversa automaticamente tutti i file nella cartella di addestramento che possono essere decodificati in audio ed esegui la normalizzazione delle sezioni. ", "step2b: 使用CPU提取音高(如果模型带音高), 使用GPU提取特征(选择卡号)": "Passaggio 2b: utilizzare la CPU per estrarre il tono (se il modello ha il tono), utilizzare la GPU per estrarre le caratteristiche (selezionare l'indice GPU):", "step3: 填写训练设置, 开始训练模型和索引": "Passaggio 3: compilare le impostazioni di addestramento e avviare l'addestramento del modello e dell'indice", @@ -257,5 +257,7 @@ "[索引训练] 外部索引链接已存在：%s": "[Addestramento indice] Il collegamento esterno esiste già: %s", "[索引训练][跳过] added索引已存在：%s": "[Addestramento indice][Saltato] L’indice added esiste già: %s", "[索引训练][跳过] trained索引已存在：%s": "[Addestramento indice][Saltato] L’indice trained esiste già: %s", - "当前设备：%s | 推理精度：%s": "Dispositivo corrente: %s | Precisione di inferenza: %s" + "当前设备：%s | 推理精度：%s": "Dispositivo corrente: %s | Precisione di inferenza: %s", + "正在预热CUDA Graph": "Riscaldamento di CUDA Graph", + "CUDA Graph预热完成": "Riscaldamento di CUDA Graph completato" } diff --git a/i18n/locale/ja_JP.json b/i18n/locale/ja_JP.json index da01393..9f987fc 100644 --- a/i18n/locale/ja_JP.json +++ b/i18n/locale/ja_JP.json @@ -1,5 +1,5 @@ -{ - "人声提取激进程度": "ボーカル抽出の強度", +{ + "人声提取激进程度": "ボーカル抽出の強度", "%s → 成功": "%s → 成功", "%s运行中，请先停止该任务": "%sを実行中です。先に停止してください", "%s进程已终止": "%sプロセスを終了しました", @@ -257,5 +257,7 @@ "[索引训练] 外部索引链接已存在：%s": "[インデックス学習] 外部インデックスリンクは既に存在します：%s", "[索引训练][跳过] added索引已存在：%s": "[インデックス学習][スキップ] addedインデックスは既に存在します：%s", "[索引训练][跳过] trained索引已存在：%s": "[インデックス学習][スキップ] trainedインデックスは既に存在します：%s", - "当前设备：%s | 推理精度：%s": "現在のデバイス：%s | 推論精度：%s" + "当前设备：%s | 推理精度：%s": "現在のデバイス：%s | 推論精度：%s", + "正在预热CUDA Graph": "CUDA Graphをウォームアップしています", + "CUDA Graph预热完成": "CUDA Graphのウォームアップが完了しました" } diff --git a/i18n/locale/ko_KR.json b/i18n/locale/ko_KR.json index b4b7b5c..2c6c914 100644 --- a/i18n/locale/ko_KR.json +++ b/i18n/locale/ko_KR.json @@ -1,5 +1,5 @@ -{ - "人声提取激进程度": "보컬 추출 강도", +{ + "人声提取激进程度": "보컬 추출 강도", "%s → 成功": "%s → 성공", "%s运行中，请先停止该任务": "%s이(가) 실행 중입니다. 먼저 중지하세요", "%s进程已终止": "%s 프로세스가 종료되었습니다", @@ -257,5 +257,7 @@ "[索引训练] 外部索引链接已存在：%s": "[인덱스 학습] 외부 인덱스 링크가 이미 있습니다: %s", "[索引训练][跳过] added索引已存在：%s": "[인덱스 학습][건너뜀] added 인덱스가 이미 있습니다: %s", "[索引训练][跳过] trained索引已存在：%s": "[인덱스 학습][건너뜀] trained 인덱스가 이미 있습니다: %s", - "当前设备：%s | 推理精度：%s": "현재 장치: %s | 추론 정밀도: %s" + "当前设备：%s | 推理精度：%s": "현재 장치: %s | 추론 정밀도: %s", + "正在预热CUDA Graph": "CUDA Graph 워밍업 중", + "CUDA Graph预热完成": "CUDA Graph 워밍업 완료" } diff --git a/i18n/locale/pt_BR.json b/i18n/locale/pt_BR.json index c4885a9..fa4900e 100644 --- a/i18n/locale/pt_BR.json +++ b/i18n/locale/pt_BR.json @@ -1,5 +1,5 @@ -{ - "人声提取激进程度": "Intensidade da extração vocal", +{ + "人声提取激进程度": "Intensidade da extração vocal", "%s → 成功": "%s → Sucesso", "%s运行中，请先停止该任务": "%s está em execução; interrompa antes de iniciar outra tarefa", "%s进程已终止": "O processo %s foi encerrado", @@ -54,7 +54,7 @@ "rmvpe卡号配置：以-分隔输入使用的不同进程卡号,例如0-0-1使用在卡0上跑2个进程并在卡1上跑1个进程": "Configuração do número do cartão rmvpe: Use - para separar os números dos cartões de entrada de diferentes processos. Por exemplo, 0-0-1 é usado para executar 2 processos no cartão 0 e 1 processo no cartão 1.", "step1: 填写实验配置. 实验数据放在logs下, 每个实验一个文件夹, 需手工输入实验名路径, 内含实验配置, 日志, 训练得到的模型文件. ": "Etapa 1: Preencha a configuração experimental. Os dados experimentais são armazenados na pasta 'logs', com cada experimento tendo uma pasta separada. Digite manualmente o caminho do nome do experimento, que contém a configuração experimental, os logs e os arquivos de modelo treinados.", "step1:正在处理数据": "Etapa 1: Processamento de dados", - "step2:正在提取音高&正在提取特征": "step2:Extração de tom e características", + "step2:正在提取音高&正在提取特征": "step2:Extração de tom e características", "step2a: 自动遍历训练文件夹下所有可解码成音频的文件并进行切片归一化, 在实验目录下生成2个wav文件夹; 暂时只支持单人训练. ": "Etapa 2a: Percorra automaticamente todos os arquivos na pasta de treinamento que podem ser decodificados em áudio e execute a normalização da fatia. Gera 2 pastas wav no diretório do experimento. Atualmente, apenas o treinamento de um único cantor/palestrante é suportado.", "step2b: 使用CPU提取音高(如果模型带音高), 使用GPU提取特征(选择卡号)": "Etapa 2b: Use a CPU para extrair o tom (se o modelo tiver tom), use a GPU para extrair recursos (selecione o índice da GPU):", "step3: 填写训练设置, 开始训练模型和索引": "Etapa 3: Preencha as configurações de treinamento e comece a treinar o modelo e o Index", @@ -257,5 +257,7 @@ "[索引训练] 外部索引链接已存在：%s": "[Treinamento de índice] O link externo já existe: %s", "[索引训练][跳过] added索引已存在：%s": "[Treinamento de índice][Ignorado] O índice added já existe: %s", "[索引训练][跳过] trained索引已存在：%s": "[Treinamento de índice][Ignorado] O índice trained já existe: %s", - "当前设备：%s | 推理精度：%s": "Dispositivo atual: %s | Precisão de inferência: %s" + "当前设备：%s | 推理精度：%s": "Dispositivo atual: %s | Precisão de inferência: %s", + "正在预热CUDA Graph": "Aquecendo o CUDA Graph", + "CUDA Graph预热完成": "Aquecimento do CUDA Graph concluído" } diff --git a/i18n/locale/ru_RU.json b/i18n/locale/ru_RU.json index c110a32..1453fcc 100644 --- a/i18n/locale/ru_RU.json +++ b/i18n/locale/ru_RU.json @@ -1,5 +1,5 @@ -{ - "人声提取激进程度": "Интенсивность выделения вокала", +{ + "人声提取激进程度": "Интенсивность выделения вокала", "%s → 成功": "%s → Успешно", "%s运行中，请先停止该任务": "%s уже выполняется; сначала остановите эту задачу", "%s进程已终止": "Процесс %s завершён", @@ -54,7 +54,7 @@ "rmvpe卡号配置：以-分隔输入使用的不同进程卡号,例如0-0-1使用在卡0上跑2个进程并在卡1上跑1个进程": "Введите номера графических процессоров, разделенные символом «-», например, 0-0-1, чтобы запустить два процесса на GPU 0 и один процесс на GPU 1:", "step1: 填写实验配置. 实验数据放在logs下, 每个实验一个文件夹, 需手工输入实验名路径, 内含实验配置, 日志, 训练得到的模型文件. ": "Шаг 1. Конфигурирование модели. Данные обучения модели сохраняются в папку 'logs', и для каждой модели создаётся отдельная папка. Введите вручную путь к настройкам для модели, в которой находятся логи и тренировочные файлы.", "step1:正在处理数据": "Шаг 1. Переработка данных", - "step2:正在提取音高&正在提取特征": "step2:Извлечение высоты тона и признаков", + "step2:正在提取音高&正在提取特征": "step2:Извлечение высоты тона и признаков", "step2a: 自动遍历训练文件夹下所有可解码成音频的文件并进行切片归一化, 在实验目录下生成2个wav文件夹; 暂时只支持单人训练. ": "Шаг 2А. Автоматическая обработка исходных аудиозаписей для обучения и выполнение нормализации среза. Создаст 2 папки wav в папке модели. В данный момент поддерживается обучение только на одноголосных записях.", "step2b: 使用CPU提取音高(如果模型带音高), 使用GPU提取特征(选择卡号)": "Шаг 2Б. Оценка и извлечение тональности в аудиофайлах с помощью процессора (если включена поддержка изменения высоты звука), извлечение черт с помощью GPU (выберите номер GPU):", "step3: 填写训练设置, 开始训练模型和索引": "Шаг 3. Заполнение дополнительных настроек обучения и запуск обучения модели и индекса", @@ -257,5 +257,7 @@ "[索引训练] 外部索引链接已存在：%s": "[Обучение индекса] Внешняя ссылка уже существует: %s", "[索引训练][跳过] added索引已存在：%s": "[Обучение индекса][Пропущено] Индекс added уже существует: %s", "[索引训练][跳过] trained索引已存在：%s": "[Обучение индекса][Пропущено] Индекс trained уже существует: %s", - "当前设备：%s | 推理精度：%s": "Текущее устройство: %s | Точность вывода: %s" + "当前设备：%s | 推理精度：%s": "Текущее устройство: %s | Точность вывода: %s", + "正在预热CUDA Graph": "Прогрев CUDA Graph", + "CUDA Graph预热完成": "Прогрев CUDA Graph завершён" } diff --git a/i18n/locale/tr_TR.json b/i18n/locale/tr_TR.json index a7fb23c..f512701 100644 --- a/i18n/locale/tr_TR.json +++ b/i18n/locale/tr_TR.json @@ -1,5 +1,5 @@ -{ - "人声提取激进程度": "Vokal ayırma yoğunluğu", +{ + "人声提取激进程度": "Vokal ayırma yoğunluğu", "%s → 成功": "%s → Başarılı", "%s运行中，请先停止该任务": "%s çalışıyor; başka bir görev başlatmadan önce durdurun", "%s进程已终止": "%s işlemi sonlandırıldı", @@ -54,7 +54,7 @@ "rmvpe卡号配置：以-分隔输入使用的不同进程卡号,例如0-0-1使用在卡0上跑2个进程并在卡1上跑1个进程": "RMVPE GPU yapılandırmasını tirelerle ayırın; örneğin 0-0-1, GPU 0'da iki ve GPU 1'de bir işlem çalıştırır", "step1: 填写实验配置. 实验数据放在logs下, 每个实验一个文件夹, 需手工输入实验名路径, 内含实验配置, 日志, 训练得到的模型文件. ": "Adım 1: Deneysel yapılandırmayı doldurun. Deneysel veriler 'logs' klasöründe saklanır ve her bir deney için ayrı bir klasör vardır. Deneysel adı yolu manuel olarak girin; bu yol, deneysel yapılandırmayı, günlükleri ve eğitilmiş model dosyalarını içerir.", "step1:正在处理数据": "Adım 1: Veri işleme", - "step2:正在提取音高&正在提取特征": "step2:Perde ve özellik çıkarımı", + "step2:正在提取音高&正在提取特征": "step2:Perde ve özellik çıkarımı", "step2a: 自动遍历训练文件夹下所有可解码成音频的文件并进行切片归一化, 在实验目录下生成2个wav文件夹; 暂时只支持单人训练. ": "Adım 2a: Eğitim klasöründe ses dosyalarını otomatik olarak gezinerek dilimleme normalizasyonu yapın. Deney dizini içinde 2 wav klasörü oluşturur. Şu anda sadece tek kişilik eğitim desteklenmektedir.", "step2b: 使用CPU提取音高(如果模型带音高), 使用GPU提取特征(选择卡号)": "Adım 2b: Ses yüksekliği (Pitch) çıkartmak için CPU kullanın (eğer model ses yüksekliği içeriyorsa), özellikleri çıkartmak için GPU kullanın (GPU indeksini seçin):", "step3: 填写训练设置, 开始训练模型和索引": "Adım 3: Eğitim ayarlarını doldurun ve modeli ve dizini eğitmeye başlayın", @@ -257,5 +257,7 @@ "[索引训练] 外部索引链接已存在：%s": "[Dizin eğitimi] Harici dizin bağlantısı zaten var: %s", "[索引训练][跳过] added索引已存在：%s": "[Dizin eğitimi][Atlandı] added dizini zaten var: %s", "[索引训练][跳过] trained索引已存在：%s": "[Dizin eğitimi][Atlandı] trained dizini zaten var: %s", - "当前设备：%s | 推理精度：%s": "Geçerli cihaz: %s | Çıkarım hassasiyeti: %s" + "当前设备：%s | 推理精度：%s": "Geçerli cihaz: %s | Çıkarım hassasiyeti: %s", + "正在预热CUDA Graph": "CUDA Graph ısınıyor", + "CUDA Graph预热完成": "CUDA Graph ısınması tamamlandı" } diff --git a/i18n/locale/zh_CN.json b/i18n/locale/zh_CN.json index f64bbb3..7658edb 100644 --- a/i18n/locale/zh_CN.json +++ b/i18n/locale/zh_CN.json @@ -1,5 +1,5 @@ -{ - "人声提取激进程度": "人声提取激进程度", +{ + "人声提取激进程度": "人声提取激进程度", "%s → 成功": "%s → 成功", "%s运行中，请先停止该任务": "%s运行中，请先停止该任务", "%s进程已终止": "%s进程已终止", @@ -257,5 +257,7 @@ "[索引训练] 外部索引链接已存在：%s": "[索引训练] 外部索引链接已存在：%s", "[索引训练][跳过] added索引已存在：%s": "[索引训练][跳过] added索引已存在：%s", "[索引训练][跳过] trained索引已存在：%s": "[索引训练][跳过] trained索引已存在：%s", - "当前设备：%s | 推理精度：%s": "当前设备：%s | 推理精度：%s" + "当前设备：%s | 推理精度：%s": "当前设备：%s | 推理精度：%s", + "正在预热CUDA Graph": "正在预热CUDA Graph", + "CUDA Graph预热完成": "CUDA Graph预热完成" } diff --git a/i18n/locale/zh_HK.json b/i18n/locale/zh_HK.json index cac4134..c451011 100644 --- a/i18n/locale/zh_HK.json +++ b/i18n/locale/zh_HK.json @@ -1,5 +1,5 @@ -{ - "人声提取激进程度": "人聲提取激進程度", +{ + "人声提取激进程度": "人聲提取激進程度", "%s → 成功": "%s → 成功", "%s运行中，请先停止该任务": "%s运行中，请先停止该任务", "%s进程已终止": "%s進程已终止", @@ -257,5 +257,7 @@ "[索引训练] 外部索引链接已存在：%s": "[索引訓練] 外部索引連結已存在：%s", "[索引训练][跳过] added索引已存在：%s": "[索引訓練][跳過] added索引已存在：%s", "[索引训练][跳过] trained索引已存在：%s": "[索引訓練][跳過] trained索引已存在：%s", - "当前设备：%s | 推理精度：%s": "目前裝置：%s | 推理精度：%s" + "当前设备：%s | 推理精度：%s": "目前裝置：%s | 推理精度：%s", + "正在预热CUDA Graph": "正在預熱 CUDA Graph", + "CUDA Graph预热完成": "CUDA Graph 預熱完成" } diff --git a/i18n/locale/zh_SG.json b/i18n/locale/zh_SG.json index 57e46ab..d3f500c 100644 --- a/i18n/locale/zh_SG.json +++ b/i18n/locale/zh_SG.json @@ -1,5 +1,5 @@ -{ - "人声提取激进程度": "人声提取激进程度", +{ + "人声提取激进程度": "人声提取激进程度", "%s → 成功": "%s → 成功", "%s运行中，请先停止该任务": "%s运行中，请先停止该任务", "%s进程已终止": "%s进程已终止", @@ -257,5 +257,7 @@ "[索引训练] 外部索引链接已存在：%s": "[索引训练] 外部索引链接已存在：%s", "[索引训练][跳过] added索引已存在：%s": "[索引训练][跳过] added索引已存在：%s", "[索引训练][跳过] trained索引已存在：%s": "[索引训练][跳过] trained索引已存在：%s", - "当前设备：%s | 推理精度：%s": "当前设备：%s | 推理精度：%s" + "当前设备：%s | 推理精度：%s": "当前设备：%s | 推理精度：%s", + "正在预热CUDA Graph": "正在预热CUDA Graph", + "CUDA Graph预热完成": "CUDA Graph预热完成" } diff --git a/i18n/locale/zh_TW.json b/i18n/locale/zh_TW.json index db3c5cb..af037dd 100644 --- a/i18n/locale/zh_TW.json +++ b/i18n/locale/zh_TW.json @@ -1,5 +1,5 @@ -{ - "人声提取激进程度": "人聲提取激進程度", +{ + "人声提取激进程度": "人聲提取激進程度", "%s → 成功": "%s → 成功", "%s运行中，请先停止该任务": "%s运行中，请先停止该任务", "%s进程已终止": "%s進程已终止", @@ -257,5 +257,7 @@ "[索引训练] 外部索引链接已存在：%s": "[索引訓練] 外部索引連結已存在：%s", "[索引训练][跳过] added索引已存在：%s": "[索引訓練][跳過] added索引已存在：%s", "[索引训练][跳过] trained索引已存在：%s": "[索引訓練][跳過] trained索引已存在：%s", - "当前设备：%s | 推理精度：%s": "目前裝置：%s | 推論精度：%s" + "当前设备：%s | 推理精度：%s": "目前裝置：%s | 推論精度：%s", + "正在预热CUDA Graph": "正在預熱 CUDA Graph", + "CUDA Graph预热完成": "CUDA Graph 預熱完成" } diff --git a/infer/fcpe.py b/infer/fcpe.py index c99d7f9..1d4f34c 100644 --- a/infer/fcpe.py +++ b/infer/fcpe.py @@ -1,5 +1,7 @@ import torch +from tools.cuda_graph import cuda_graph_enabled, run_cuda_graph + def _is_directml_device(device): """Return whether *device* is the PrivateUse1 device registered by DirectML.""" @@ -35,6 +37,46 @@ class FCPEInfer: self.infer_model.model.to(device).eval() else: self.infer_model = spawn_bundled_infer_model(device) + if getattr(device, "type", None) == "cuda" or str(device).startswith( + "cuda" + ): + # torchfcpe creates this tensor on CPU and copies it to CUDA in + # every local-argmax decode. Host-to-device copies are forbidden + # during CUDA Graph capture, so keep the immutable offsets on the + # target device and use the graph-safe equivalent decoder below. + self.local_offsets = torch.arange( + 9, device=device, dtype=torch.long + ).view(1, 1, 9) + + def _graphable_model_infer(self, mel, decoder_mode, threshold): + """Run FCPE network and an exactly equivalent capture-safe decoder.""" + model = self.infer_model.model + latent = model(mel) + batch, frames, _ = latent.shape + cents = model.cent_table[None, None, :].expand(batch, frames, -1) + + if decoder_mode == "argmax": + confidence = torch.max(latent, dim=-1, keepdim=True).values + decoded = torch.sum(cents * latent, dim=-1, keepdim=True) / torch.sum( + latent, dim=-1, keepdim=True + ) + elif decoder_mode == "local_argmax": + confidence, max_index = torch.max(latent, dim=-1, keepdim=True) + local_index = self.local_offsets + (max_index - 4) + local_index = local_index.clamp(0, model.out_dims - 1) + local_cents = torch.gather(cents, -1, local_index) + local_latent = torch.gather(latent, -1, local_index) + decoded = torch.sum( + local_cents * local_latent, dim=-1, keepdim=True + ) / torch.sum(local_latent, dim=-1, keepdim=True) + else: + raise ValueError("Unknown FCPE decoder mode: %s" % decoder_mode) + + # Match torchfcpe's masking and cent-to-Hz formulas operation-for-operation. + confidence_mask = torch.ones_like(confidence) + confidence_mask.masked_fill_(confidence <= threshold, float("-inf")) + decoded = decoded * confidence_mask + return 10.0 * torch.pow(2.0, decoded / 1200.0) def _decode_on_cpu(self, latent, decoder_mode, threshold): """Decode DML network logits on CPU with torchfcpe's exact formulas. @@ -79,11 +121,33 @@ class FCPEInfer: threshold=0.006, ): if not self.is_directml: - return self.infer_model.infer( + wav = wav.to(self.device) + if cuda_graph_enabled(wav.device): + # Wav2MelModule contains tensor-dependent Python conditionals and + # cannot be captured. Running it eagerly also creates/caches its + # STFT window before the graph. The much larger FCPE neural net + # and decoder form the stable-shape CUDA Graph boundary. + mel = self.infer_model.wav2mel(wav, sr) + return run_cuda_graph( + self.infer_model.model, + "fcpe-core-%s-%s" % (decoder_mode, threshold), + lambda input_mel: self._graphable_model_infer( + input_mel, + decoder_mode, + threshold, + ), + mel, + ) + return run_cuda_graph( + self.infer_model, + "fcpe-%s-%s-%s" % (sr, decoder_mode, threshold), + lambda input_wav: self.infer_model.infer( + input_wav, + sr=sr, + decoder_mode=decoder_mode, + threshold=threshold, + ), wav, - sr=sr, - decoder_mode=decoder_mode, - threshold=threshold, ) wav_cpu = wav.detach().to(device="cpu", dtype=torch.float32) diff --git a/infer/hubert.py b/infer/hubert.py index 128a628..6e2bd16 100644 --- a/infer/hubert.py +++ b/infer/hubert.py @@ -4,7 +4,9 @@ from pathlib import Path import torch from torch import nn -from transformers import AutoFeatureExtractor, HubertModel +from transformers import AutoFeatureExtractor, HubertModel + +from tools.cuda_graph import run_cuda_graph logger = logging.getLogger(__name__) @@ -63,7 +65,7 @@ def hubert_audio_requires_normalization(): return bool(feature_extractor.do_normalize) -def extract_hubert_features(model, source, version, padding_mask=None): +def extract_hubert_features(model, source, version, padding_mask=None): """Return the RVC v1 (256-D) or v2 (768-D) HuBERT representation. Transformers hidden_states[N] is numerically equivalent to the source checkpoint's @@ -77,20 +79,49 @@ def extract_hubert_features(model, source, version, padding_mask=None): if padding_mask is not None and bool(torch.any(padding_mask).item()): attention_mask = (~padding_mask.bool()).long() - if version == "v1": - outputs = model( - input_values=source, - attention_mask=attention_mask, - output_hidden_states=True, - return_dict=True, - ) - features = outputs.hidden_states[9] - return model.final_proj(features) - - outputs = model( - input_values=source, - attention_mask=attention_mask, - output_hidden_states=False, - return_dict=True, - ) - return outputs.last_hidden_state + if version == "v1": + if attention_mask is None: + def forward(input_values): + outputs = model( + input_values=input_values, + attention_mask=None, + output_hidden_states=True, + return_dict=True, + ) + return model.final_proj(outputs.hidden_states[9]) + + return run_cuda_graph(model, "hubert-v1-no-mask", forward, source) + + def forward(input_values, mask): + outputs = model( + input_values=input_values, + attention_mask=mask, + output_hidden_states=True, + return_dict=True, + ) + return model.final_proj(outputs.hidden_states[9]) + + return run_cuda_graph( + model, "hubert-v1-mask", forward, source, attention_mask + ) + + if attention_mask is None: + def forward(input_values): + return model( + input_values=input_values, + attention_mask=None, + output_hidden_states=False, + return_dict=True, + ).last_hidden_state + + return run_cuda_graph(model, "hubert-v2-no-mask", forward, source) + + def forward(input_values, mask): + return model( + input_values=input_values, + attention_mask=mask, + output_hidden_states=False, + return_dict=True, + ).last_hidden_state + + return run_cuda_graph(model, "hubert-v2-mask", forward, source, attention_mask) diff --git a/infer/module/models.py b/infer/module/models.py index bb668fe..be2a29e 100644 --- a/infer/module/models.py +++ b/infer/module/models.py @@ -67,8 +67,11 @@ class TextEncoder(nn.Module): ) x = self.encoder(x * x_mask, x_mask) if skip_head is not None: - assert isinstance(skip_head, torch.Tensor) - head = int(skip_head.item()) + head = ( + int(skip_head.item()) + if isinstance(skip_head, torch.Tensor) + else int(skip_head) + ) x = x[:, :, head:] x_mask = x_mask[:, :, head:] stats = self.proj(x) * x_mask @@ -231,8 +234,7 @@ class Generator(torch.nn.Module): n_res = None, ): if n_res is not None: - assert isinstance(n_res, torch.Tensor) - n = int(n_res.item()) + n = int(n_res.item()) if isinstance(n_res, torch.Tensor) else int(n_res) if n != x.shape[-1]: x = F.interpolate(x, size=n, mode="linear") x = self.conv_pre(x) @@ -501,8 +503,7 @@ class GeneratorNSF(torch.nn.Module): har_source, noi_source, uv = self.m_source(f0, self.upp) har_source = har_source.transpose(1, 2) if n_res is not None: - assert isinstance(n_res, torch.Tensor) - n = int(n_res.item()) + n = int(n_res.item()) if isinstance(n_res, torch.Tensor) else int(n_res) if n * self.upp != har_source.shape[-1]: har_source = F.interpolate(har_source, size=n * self.upp, mode="linear") if n != x.shape[-1]: @@ -674,12 +675,14 @@ class SynthesizerTrnMs256NSFsid(nn.Module): ): g = self.emb_g(sid).unsqueeze(-1) if skip_head is not None and return_length is not None: - assert isinstance(skip_head, torch.Tensor) - assert isinstance(return_length, torch.Tensor) - head = int(skip_head.item()) - length = int(return_length.item()) - flow_head = torch.clamp(skip_head - 24, min=0) - dec_head = head - int(flow_head.item()) + head = int(skip_head.item()) if isinstance(skip_head, torch.Tensor) else int(skip_head) + length = ( + int(return_length.item()) + if isinstance(return_length, torch.Tensor) + else int(return_length) + ) + flow_head = max(head - 24, 0) + dec_head = head - flow_head m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths, flow_head) z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask z = self.flow(z_p, x_mask, g=g, reverse=True) @@ -862,12 +865,14 @@ class SynthesizerTrnMs256NSFsid_nono(nn.Module): ): g = self.emb_g(sid).unsqueeze(-1) if skip_head is not None and return_length is not None: - assert isinstance(skip_head, torch.Tensor) - assert isinstance(return_length, torch.Tensor) - head = int(skip_head.item()) - length = int(return_length.item()) - flow_head = torch.clamp(skip_head - 24, min=0) - dec_head = head - int(flow_head.item()) + head = int(skip_head.item()) if isinstance(skip_head, torch.Tensor) else int(skip_head) + length = ( + int(return_length.item()) + if isinstance(return_length, torch.Tensor) + else int(return_length) + ) + flow_head = max(head - 24, 0) + dec_head = head - flow_head m_p, logs_p, x_mask = self.enc_p(phone, None, phone_lengths, flow_head) z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask z = self.flow(z_p, x_mask, g=g, reverse=True) diff --git a/infer/rmvpe.py b/infer/rmvpe.py index cccb8d0..f0496bd 100644 --- a/infer/rmvpe.py +++ b/infer/rmvpe.py @@ -6,7 +6,9 @@ import torch import torch.nn as nn import torch.nn.functional as F from librosa.util import normalize, pad_center, tiny -from scipy.signal import get_window +from scipy.signal import get_window + +from tools.cuda_graph import run_cuda_graph import logging @@ -536,7 +538,7 @@ class RMVPE: cents_mapping = 20 * np.arange(360) + 1997.3794084376191 self.cents_mapping = np.pad(cents_mapping, (4, 4)) # 368 - def mel2hidden(self, mel): + def mel2hidden(self, mel): with torch.no_grad(): n_frames = mel.shape[-1] n_pad = 32 * ((n_frames - 1) // 32 + 1) - n_frames @@ -549,10 +551,30 @@ class RMVPE: [onnx_outputs_names], input_feed={onnx_input_name: mel.cpu().numpy()}, )[0] - else: - mel = mel.half() if self.is_half else mel.float() - hidden = self.model(mel) - return hidden[:, :n_frames] + else: + mel = mel.half() if self.is_half else mel.float() + hidden = run_cuda_graph( + self.model, + "rmvpe-network", + lambda input_mel: self.model(input_mel), + mel, + ) + return hidden[:, :n_frames] + + def extract_mel(self, audio, center=True): + if not torch.is_tensor(audio): + audio = torch.from_numpy(audio) + audio = audio.float().to(self.device) + if audio.dim() == 1: + audio = audio.unsqueeze(0) + if "privateuseone" in str(self.device): + return self.mel_extractor(audio, center=center) + return run_cuda_graph( + self.mel_extractor, + "rmvpe-mel-center-%s" % int(bool(center)), + lambda input_audio: self.mel_extractor(input_audio, center=center), + audio, + ) def decode(self, hidden, thred=0.03): cents_pred = self.to_local_average_cents(hidden, thred=thred) @@ -564,11 +586,7 @@ class RMVPE: def infer_from_audio(self, audio, thred=0.03): # torch.cuda.synchronize() # t0 = ttime() - if not torch.is_tensor(audio): - audio = torch.from_numpy(audio) - mel = self.mel_extractor( - audio.float().to(self.device).unsqueeze(0), center=True - ) + mel = self.extract_mel(audio, center=True) # print(123123123,mel.device.type) # torch.cuda.synchronize() # t1 = ttime() diff --git a/infer/rtrvc.py b/infer/rtrvc.py index 4fced9f..ecd6b31 100644 --- a/infer/rtrvc.py +++ b/infer/rtrvc.py @@ -10,6 +10,7 @@ from torchaudio.transforms import Resample from infer.hubert import extract_hubert_features, load_hubert_model from i18n.i18n import I18nAuto +from tools.cuda_graph import run_cuda_graph i18n = I18nAuto() @@ -93,6 +94,7 @@ class RVC: self.cache_pitchf = torch.zeros( 1024, device=self.device, dtype=torch.float32 ) + self.infer_count = 0 self.resample_kernel = {} @@ -179,11 +181,9 @@ class RVC: if len(f0) < p_len: f0 = np.pad(f0, (0, p_len - len(f0))) f0 = f0[:p_len] - try: - uv = f0 == 0 + uv = f0 == 0 + if np.any(~uv): f0[uv] = np.interp(np.where(uv)[0], np.where(~uv)[0], f0[~uv]) - except Exception: - traceback.print_exc() f0 *= pow(2, f0_up_key / 12) return self.get_f0_post(f0) @@ -198,11 +198,9 @@ class RVC: device=self.device, ) f0 = self.model_rmvpe.infer_from_audio(x, thred=0.03) - try: - uv = f0 == 0 + uv = f0 == 0 + if np.any(~uv): f0[uv] = np.interp(np.where(uv)[0], np.where(~uv)[0], f0[~uv]) - except Exception: - traceback.print_exc() f0 *= pow(2, f0_up_key / 12) return self.get_f0_post(f0) @@ -218,11 +216,9 @@ class RVC: decoder_mode="local_argmax", threshold=0.006, ).squeeze().detach().cpu().numpy() - try: - uv = f0 == 0 + uv = f0 == 0 + if np.any(~uv): f0[uv] = np.interp(np.where(uv)[0], np.where(~uv)[0], f0[~uv]) - except Exception: - traceback.print_exc() f0 *= pow(2, f0_up_key / 12) return self.get_f0_post(f0) @@ -234,6 +230,8 @@ class RVC: return_length, f0method, ) : + report_status = self.infer_count < 3 or self.infer_count % 100 == 0 + self.infer_count += 1 t1 = ttime() with torch.no_grad(): if self.config.is_half: @@ -271,7 +269,8 @@ class RVC: i18n("索引无效：必须使用added_xxxx.index，不能使用trained_xxxx.index") ) else: - printt(i18n("索引检索失败或未启用")) + if report_status: + printt(i18n("索引检索失败或未启用")) except Exception: traceback.print_exc() printt(i18n("索引检索失败")) @@ -298,26 +297,49 @@ class RVC: t4 = ttime() feats = F.interpolate(feats.permute(0, 2, 1), scale_factor=2).permute(0, 2, 1) feats = feats[:, :p_len, :] - p_len = torch.LongTensor([p_len]).to(self.device) + p_len_tensor = torch.LongTensor([p_len]).to(self.device) sid = torch.LongTensor([0]).to(self.device) - skip_head = torch.LongTensor([skip_head]) - return_length2 = torch.LongTensor([return_length2]) - return_length = torch.LongTensor([return_length]) + skip_head_value = int(skip_head) + return_length_value = int(return_length) + return_length2_value = int(return_length2) with torch.no_grad(): if self.if_f0 == 1: - infered_audio, _, _ = self.net_g.infer( + infered_audio = run_cuda_graph( + self.net_g, + "rvc-realtime-f0-%s-%s-%s" + % (skip_head_value, return_length_value, return_length2_value), + lambda phone, lengths, coarse, continuous, speaker: self.net_g.infer( + phone, + lengths, + coarse, + continuous, + speaker, + skip_head_value, + return_length_value, + return_length2_value, + )[0], feats, - p_len, + p_len_tensor, cache_pitch, cache_pitchf, sid, - skip_head, - return_length, - return_length2, ) else: - infered_audio, _, _ = self.net_g.infer( - feats, p_len, sid, skip_head, return_length, return_length2 + infered_audio = run_cuda_graph( + self.net_g, + "rvc-realtime-no-f0-%s-%s-%s" + % (skip_head_value, return_length_value, return_length2_value), + lambda phone, lengths, speaker: self.net_g.infer( + phone, + lengths, + speaker, + skip_head_value, + return_length_value, + return_length2_value, + )[0], + feats, + p_len_tensor, + sid, ) infered_audio = infered_audio.squeeze(1).float() upp_res = int(np.floor(factor * self.tgt_sr // 100)) @@ -332,11 +354,12 @@ class RVC: infered_audio[:, : return_length * upp_res] ) t5 = ttime() - printt( - i18n("耗时：特征=%.3f秒，索引=%.3f秒，音高=%.3f秒，模型=%.3f秒"), - t2 - t1, - t3 - t2, - t4 - t3, - t5 - t4, - ) + if report_status: + printt( + i18n("耗时：特征=%.3f秒，索引=%.3f秒，音高=%.3f秒，模型=%.3f秒"), + t2 - t1, + t3 - t2, + t4 - t3, + t5 - t4, + ) return infered_audio.squeeze() diff --git a/infer/vc/modules.py b/infer/vc/modules.py index 54d62ac..e198e1a 100644 --- a/infer/vc/modules.py +++ b/infer/vc/modules.py @@ -19,6 +19,7 @@ from infer.vc.pipeline import Pipeline from infer.vc.utils import * from i18n.i18n import I18nAuto from tools.progress import batch_status, should_report +from tools.cuda_graph import clear_cuda_graph_cache i18n = I18nAuto() @@ -68,6 +69,8 @@ class VC: self.hubert_model is not None ): # 考虑到轮询, 需要加个判断看是否 sid 是由有模型切换到无模型的 logger.info(i18n("清理模型缓存")) + clear_cuda_graph_cache(self.net_g) + clear_cuda_graph_cache(self.hubert_model) del (self.net_g, self.n_spk, self.hubert_model, self.tgt_sr) # ,cpt self.hubert_model = self.net_g = self.n_spk = self.hubert_model = ( self.tgt_sr @@ -112,6 +115,9 @@ class VC: person = f'{os.getenv("weight_root")}/{sid}' logger.info("%s: %s", i18n("正在加载模型"), person) + if self.net_g is not None: + clear_cuda_graph_cache(self.net_g) + self.cpt = torch.load(person, map_location="cpu") self.tgt_sr = self.cpt["config"][-1] self.cpt["config"][-3] = self.cpt["weight"]["emb_g.weight"].shape[0] # n_spk diff --git a/infer/vc/pipeline.py b/infer/vc/pipeline.py index 6ba32dc..2aa3235 100644 --- a/infer/vc/pipeline.py +++ b/infer/vc/pipeline.py @@ -15,6 +15,7 @@ import torch.nn.functional as F from scipy import signal from infer.hubert import extract_hubert_features +from tools.cuda_graph import cuda_graph_enabled, run_cuda_graph bh, ah = signal.butter(N=5, Wn=48, btype="high", fs=16000) @@ -217,11 +218,34 @@ class Pipeline(object): p_len = torch.tensor([p_len], device=self.device).long() with torch.no_grad(): hasp = pitch is not None and pitchf is not None - arg = (feats, p_len, pitch, pitchf, sid) if hasp else (feats, p_len, sid) - audio1 = (net_g.infer(*arg)[0][0, 0]).data.cpu().float().numpy() - del hasp, arg + if hasp: + synthesized = run_cuda_graph( + net_g, + "rvc-synth-f0", + lambda phone, lengths, coarse, continuous, speaker: net_g.infer( + phone, lengths, coarse, continuous, speaker + )[0], + feats, + p_len, + pitch, + pitchf, + sid, + ) + else: + synthesized = run_cuda_graph( + net_g, + "rvc-synth-no-f0", + lambda phone, lengths, speaker: net_g.infer( + phone, lengths, speaker + )[0], + feats, + p_len, + sid, + ) + audio1 = synthesized[0, 0].data.cpu().float().numpy() + del hasp, synthesized del feats, p_len, padding_mask - if torch.cuda.is_available(): + if torch.cuda.is_available() and not cuda_graph_enabled(self.device): torch.cuda.empty_cache() t2 = ttime() times[0] += t1 - t0 @@ -381,6 +405,6 @@ class Pipeline(object): max_int16 /= audio_max audio_opt = (audio_opt * max_int16).astype(np.int16) del pitch, pitchf, sid - if torch.cuda.is_available(): + if torch.cuda.is_available() and not cuda_graph_enabled(self.device): torch.cuda.empty_cache() return audio_opt diff --git a/realtime_gui.py b/realtime_gui.py index 2efe967..c8ff772 100644 --- a/realtime_gui.py +++ b/realtime_gui.py @@ -36,9 +36,10 @@ if __name__ == "__main__": import torch.nn.functional as F import torchaudio.transforms as tat + from configs.config import Config from infer import rtrvc as rvc_for_realtime - from i18n.i18n import I18nAuto - from configs.config import Config + from i18n.i18n import I18nAuto + from tools.cuda_graph import cuda_graph_enabled, run_cuda_graph i18n = I18nAuto() @@ -64,9 +65,10 @@ if __name__ == "__main__": self.sg_output_device = "" class GUI: - def __init__(self) : - self.gui_config = GUIConfig() - self.config = Config() + def __init__(self) : + self.gui_config = GUIConfig() + self.config = Config() + printt("RVC_CUDA_GRAPH=%s", os.environ.get("RVC_CUDA_GRAPH", "0")) self.function = "vc" self.delay_time = 0 self.hostapis = None @@ -629,9 +631,16 @@ if __name__ == "__main__": dtype=torch.float32, ) self.rms_buffer = np.zeros(4 * self.zc, dtype="float32") - self.sola_buffer = torch.zeros( - self.sola_buffer_frame, device=self.config.device, dtype=torch.float32 - ) + self.sola_buffer = torch.zeros( + self.sola_buffer_frame, device=self.config.device, dtype=torch.float32 + ) + self.sola_den_kernel = torch.ones( + 1, + 1, + self.sola_buffer_frame, + device=self.config.device, + dtype=torch.float32, + ) self.nr_buffer = self.sola_buffer.clone() self.output_buffer = self.input_wav.clone() self.skip_head = self.extra_frame // self.zc @@ -666,12 +675,85 @@ if __name__ == "__main__": ).to(self.config.device) else: self.resampler2 = None - self.tg = TorchGate( - sr=self.gui_config.samplerate, n_fft=4 * self.zc, prop_decrease=0.9 - ).to(self.config.device) - self.start_stream() - - def start_stream(self): + self.tg = TorchGate( + sr=self.gui_config.samplerate, n_fft=4 * self.zc, prop_decrease=0.9 + ).to(self.config.device) + self.prewarm_cuda_graph() + self.start_stream() + + def prewarm_cuda_graph(self): + if not cuda_graph_enabled(self.config.device): + return + try: + printt(i18n("正在预热CUDA Graph")) + samples = self.input_wav_res.shape[0] + phase = torch.arange( + samples, device=self.config.device, dtype=torch.float32 + ) + probe = 0.05 * torch.sin(2 * np.pi * 220.0 * phase / 16000.0) + self.input_wav_res.copy_(probe) + + if self.gui_config.I_noise_reduce: + short = self.input_wav[ + -self.sola_buffer_frame - self.block_frame : + ].unsqueeze(0) + run_cuda_graph( + self.tg, + "realtime-input-noise-reduction", + lambda short_audio, full_audio: self.tg( + short_audio, full_audio + ), + short, + self.input_wav.unsqueeze(0), + ) + + resample_input = self.input_wav[-self.block_frame - 2 * self.zc :] + run_cuda_graph( + self.resampler, + "realtime-input-resample", + lambda audio: self.resampler(audio), + resample_input, + ) + + inferred = self.rvc.infer( + self.input_wav_res, + self.block_frame_16k, + self.skip_head, + self.return_length, + self.gui_config.f0method, + ) + if self.resampler2 is not None: + inferred = run_cuda_graph( + self.resampler2, + "realtime-output-resample", + lambda audio: self.resampler2(audio), + inferred, + ) + if self.gui_config.O_noise_reduce: + run_cuda_graph( + self.tg, + "realtime-output-noise-reduction", + lambda short_audio, full_audio: self.tg( + short_audio, full_audio + ), + inferred.unsqueeze(0), + self.output_buffer.unsqueeze(0), + ) + torch.cuda.synchronize(self.config.device) + printt(i18n("CUDA Graph预热完成")) + except Exception: + printt(traceback.format_exc()) + finally: + self.input_wav.zero_() + self.input_wav_denoise.zero_() + self.input_wav_res.zero_() + self.output_buffer.zero_() + self.sola_buffer.zero_() + self.nr_buffer.zero_() + self.rvc.cache_pitch.zero_() + self.rvc.cache_pitchf.zero_() + + def start_stream(self): global flag_vc if not flag_vc: flag_vc = True @@ -739,9 +821,13 @@ if __name__ == "__main__": self.block_frame : ].clone() input_wav = self.input_wav[-self.sola_buffer_frame - self.block_frame :] - input_wav = self.tg( - input_wav.unsqueeze(0), self.input_wav.unsqueeze(0) - ).squeeze(0) + input_wav = run_cuda_graph( + self.tg, + "realtime-input-noise-reduction", + lambda short, full: self.tg(short, full), + input_wav.unsqueeze(0), + self.input_wav.unsqueeze(0), + ).squeeze(0) input_wav[: self.sola_buffer_frame] *= self.fade_in_window input_wav[: self.sola_buffer_frame] += ( self.nr_buffer * self.fade_out_window @@ -750,15 +836,23 @@ if __name__ == "__main__": : self.block_frame ] self.nr_buffer[:] = input_wav[self.block_frame :] - self.input_wav_res[-self.block_frame_16k - 160 :] = self.resampler( - self.input_wav_denoise[-self.block_frame - 2 * self.zc :] - )[160:] - else: - self.input_wav_res[-160 * (indata.shape[0] // self.zc + 1) :] = ( - self.resampler(self.input_wav[-indata.shape[0] - 2 * self.zc :])[ - 160: - ] - ) + resample_input = self.input_wav_denoise[ + -self.block_frame - 2 * self.zc : + ] + self.input_wav_res[-self.block_frame_16k - 160 :] = run_cuda_graph( + self.resampler, + "realtime-input-resample", + lambda audio: self.resampler(audio), + resample_input, + )[160:] + else: + resample_input = self.input_wav[-indata.shape[0] - 2 * self.zc :] + self.input_wav_res[-160 * (indata.shape[0] // self.zc + 1) :] = run_cuda_graph( + self.resampler, + "realtime-input-resample", + lambda audio: self.resampler(audio), + resample_input, + )[160:] # infer if self.function == "vc": infer_wav = self.rvc.infer( @@ -767,9 +861,14 @@ if __name__ == "__main__": self.skip_head, self.return_length, self.gui_config.f0method, - ) - if self.resampler2 is not None: - infer_wav = self.resampler2(infer_wav) + ) + if self.resampler2 is not None: + infer_wav = run_cuda_graph( + self.resampler2, + "realtime-output-resample", + lambda audio: self.resampler2(audio), + infer_wav, + ) elif self.gui_config.I_noise_reduce: infer_wav = self.input_wav_denoise[self.extra_frame :].clone() else: @@ -780,9 +879,13 @@ if __name__ == "__main__": self.block_frame : ].clone() self.output_buffer[-self.block_frame :] = infer_wav[-self.block_frame :] - infer_wav = self.tg( - infer_wav.unsqueeze(0), self.output_buffer.unsqueeze(0) - ).squeeze(0) + infer_wav = run_cuda_graph( + self.tg, + "realtime-output-noise-reduction", + lambda short, full: self.tg(short, full), + infer_wav.unsqueeze(0), + self.output_buffer.unsqueeze(0), + ).squeeze(0) # volume envelop mixing if self.gui_config.rms_mix_rate < 1 and self.function == "vc": if self.gui_config.I_noise_reduce: @@ -813,20 +916,20 @@ if __name__ == "__main__": mode="linear", align_corners=True, )[0, 0, :-1] - rms2 = torch.max(rms2, torch.zeros_like(rms2) + 1e-3) - infer_wav *= torch.pow( - rms1 / rms2, torch.tensor(1 - self.gui_config.rms_mix_rate) - ) + rms2 = torch.max(rms2, torch.zeros_like(rms2) + 1e-3) + infer_wav *= torch.pow( + rms1 / rms2, 1.0 - self.gui_config.rms_mix_rate + ) # SOLA algorithm from https://github.com/yxlllc/DDSP-SVC conv_input = infer_wav[ None, None, : self.sola_buffer_frame + self.sola_search_frame ] cor_nom = F.conv1d(conv_input, self.sola_buffer[None, None, :]) cor_den = torch.sqrt( - F.conv1d( - conv_input**2, - torch.ones(1, 1, self.sola_buffer_frame, device=self.config.device), - ) + F.conv1d( + conv_input**2, + self.sola_den_kernel, + ) + 1e-8 ) if sys.platform == "darwin": diff --git a/tools/cuda_graph.py b/tools/cuda_graph.py new file mode 100644 index 0000000..992e7fb --- /dev/null +++ b/tools/cuda_graph.py @@ -0,0 +1,227 @@ +import logging +import os +import threading +import time +from collections import OrderedDict + +import torch + + +logger = logging.getLogger(__name__) + +ENV_NAME = "RVC_CUDA_GRAPH" +MAX_CACHE_ENV = "RVC_CUDA_GRAPH_MAX_CACHE" +_probe_lock = threading.Lock() +_probe_result = None + + +def _device_type(device): + if isinstance(device, torch.device): + return device.type + return str(device).split(":", 1)[0].lower() + + +def _cuda_device(device): + parsed = device if isinstance(device, torch.device) else torch.device(device) + if parsed.index is None: + parsed = torch.device("cuda", torch.cuda.current_device()) + return parsed + + +def _clone_output(value): + if torch.is_tensor(value): + return value.clone() + if isinstance(value, tuple): + return tuple(_clone_output(item) for item in value) + if isinstance(value, list): + return [_clone_output(item) for item in value] + if isinstance(value, dict): + return {key: _clone_output(item) for key, item in value.items()} + return value + + +def detect_cuda_graph_support(device): + if _device_type(device) != "cuda" or not torch.cuda.is_available(): + return False + if not hasattr(torch.cuda, "CUDAGraph") or not hasattr(torch.cuda, "graph"): + return False + cuda_device = _cuda_device(device) + try: + with torch.cuda.device(cuda_device): + current = torch.cuda.current_stream(cuda_device) + warmup = torch.cuda.Stream(device=cuda_device) + warmup.wait_stream(current) + with torch.cuda.stream(warmup): + probe = torch.arange(32, device=cuda_device, dtype=torch.float32) + for _ in range(3): + expected = probe.square().add_(1) + current.wait_stream(warmup) + torch.cuda.synchronize(cuda_device) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = probe.square() + 1 + probe.copy_(torch.arange(32, device=cuda_device, dtype=torch.float32)) + graph.replay() + torch.cuda.synchronize(cuda_device) + valid = torch.equal( + captured.cpu(), torch.arange(32, dtype=torch.float32).square() + 1 + ) + del captured, expected, graph, probe + return bool(valid) + except Exception: + logger.exception("CUDA Graph support probe failed on %s", cuda_device) + return False + + +def configure_cuda_graph(device): + global _probe_result + explicit = os.environ.get(ENV_NAME) + if explicit in {"0", "1"}: + if explicit == "0": + return False + if _device_type(device) != "cuda": + os.environ[ENV_NAME] = "0" + return False + with _probe_lock: + if _probe_result is None: + _probe_result = detect_cuda_graph_support(device) + os.environ[ENV_NAME] = "1" if _probe_result else "0" + return bool(_probe_result) + + +def cuda_graph_enabled(device): + return ( + os.environ.get(ENV_NAME) == "1" + and _device_type(device) == "cuda" + and torch.cuda.is_available() + ) + + +def _tensor_signature(tensor): + return ( + tuple(tensor.shape), + tuple(tensor.stride()), + str(tensor.dtype), + str(tensor.device), + bool(tensor.requires_grad), + ) + + +class _CapturedCall: + def __init__(self, function, inputs): + started = time.perf_counter() + self.lock = threading.RLock() + self.inputs = tuple(torch.empty_like(value) for value in inputs) + for static, value in zip(self.inputs, inputs): + static.copy_(value) + device = self.inputs[0].device + current = torch.cuda.current_stream(device) + warmup = torch.cuda.Stream(device=device) + warmup.wait_stream(current) + with torch.cuda.stream(warmup), torch.no_grad(): + for _ in range(3): + output = function(*self.inputs) + current.wait_stream(warmup) + torch.cuda.synchronize(device) + self.graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(self.graph), torch.no_grad(): + self.output = function(*self.inputs) + self.capture_ms = (time.perf_counter() - started) * 1000.0 + self.done_event = None + del output + + def replay(self, inputs): + with self.lock: + stream = torch.cuda.current_stream(self.inputs[0].device) + if self.done_event is not None: + stream.wait_event(self.done_event) + for static, value in zip(self.inputs, inputs): + static.copy_(value, non_blocking=True) + self.graph.replay() + output = _clone_output(self.output) + self.done_event = torch.cuda.Event(blocking=False) + self.done_event.record(stream) + return output + + +class _GraphCache: + def __init__(self): + self.entries = OrderedDict() + self.failures = set() + self.lock = threading.RLock() + self.capture_count = 0 + self.replay_count = 0 + self.fallback_count = 0 + self.eviction_count = 0 + self.capture_ms = 0.0 + + def run(self, key, function, inputs): + signature = key + tuple(_tensor_signature(value) for value in inputs) + with self.lock: + if signature in self.failures: + self.fallback_count += 1 + return function(*inputs) + entry = self.entries.get(signature) + if entry is None: + try: + entry = _CapturedCall(function, inputs) + self.entries[signature] = entry + self.capture_count += 1 + self.capture_ms += entry.capture_ms + max_entries = max(1, int(os.environ.get(MAX_CACHE_ENV, "8"))) + while len(self.entries) > max_entries: + self.entries.popitem(last=False) + self.eviction_count += 1 + except Exception: + self.failures.add(signature) + self.fallback_count += 1 + logger.exception("CUDA Graph capture failed for %s; using eager", key) + return function(*inputs) + else: + self.entries.move_to_end(signature) + output = entry.replay(inputs) + with self.lock: + self.replay_count += 1 + return output + + +def run_cuda_graph(owner, namespace, function, *inputs): + if not inputs or not cuda_graph_enabled(inputs[0].device): + return function(*inputs) + cache = getattr(owner, "_rvc_cuda_graph_cache", None) + if cache is None: + cache = _GraphCache() + setattr(owner, "_rvc_cuda_graph_cache", cache) + return cache.run((str(namespace),), function, tuple(inputs)) + + +def clear_cuda_graph_cache(owner): + cache = getattr(owner, "_rvc_cuda_graph_cache", None) + if cache is not None: + cache.entries.clear() + cache.failures.clear() + delattr(owner, "_rvc_cuda_graph_cache") + + +def get_cuda_graph_stats(owner): + cache = getattr(owner, "_rvc_cuda_graph_cache", None) + if cache is None: + return { + "entries": 0, + "failures": 0, + "captures": 0, + "replays": 0, + "fallbacks": 0, + "evictions": 0, + "capture_ms": 0.0, + } + with cache.lock: + return { + "entries": len(cache.entries), + "failures": len(cache.failures), + "captures": cache.capture_count, + "replays": cache.replay_count, + "fallbacks": cache.fallback_count, + "evictions": cache.eviction_count, + "capture_ms": cache.capture_ms, + } diff --git a/tools/uvr5/bsroformer.py b/tools/uvr5/bsroformer.py index ed595e6..ce20601 100644 --- a/tools/uvr5/bsroformer.py +++ b/tools/uvr5/bsroformer.py @@ -9,6 +9,8 @@ import soundfile as sf import torch import torch.nn as nn import yaml + +from tools.cuda_graph import run_cuda_graph from tqdm import tqdm from tools.file_io import read_text from i18n.i18n import I18nAuto @@ -178,7 +180,12 @@ class Roformer_Loader: if len(batch_data) >= batch_size or (i >= mix.shape[1]): arr = torch.stack(batch_data, dim=0) # print(23333333,arr.dtype) - x = model(arr) + x = run_cuda_graph( + model, + "uvr-bsroformer", + lambda audio: model(audio), + arr, + ) window = window_middle if i - step == 0: # First audio chunk, no fadein diff --git a/tools/uvr5/lib/utils.py b/tools/uvr5/lib/utils.py index 826b76d..cfdbea0 100644 --- a/tools/uvr5/lib/utils.py +++ b/tools/uvr5/lib/utils.py @@ -1,5 +1,6 @@ import numpy as np import torch +from tools.cuda_graph import run_cuda_graph from tqdm import tqdm @@ -34,7 +35,12 @@ def inference(X_spec, device, model, aggressiveness, data): X_mag_window = X_mag_window.half() X_mag_window = X_mag_window.to(device) - pred = model.predict(X_mag_window, aggressiveness) + pred = run_cuda_graph( + model, + "uvr-vr-%s" % repr(aggressiveness), + lambda window: model.predict(window, aggressiveness), + X_mag_window, + ) pred = pred.detach().cpu().numpy() preds.append(pred[0]) diff --git a/webui.py b/webui.py index 95ec997..77dd06d 100644 --- a/webui.py +++ b/webui.py @@ -25,6 +25,7 @@ for name in os.listdir(tmp): except Exception as error: print(str(error)) +from configs.config import Config, GPU_INDEX, GPU_INFOS, GPU_MEMORY, IS_GPU from infer.vc.modules import VC from tools.uvr5.webui import uvr from tools.file_io import read_text @@ -35,7 +36,6 @@ from train.process_ckpt import ( show_info, ) from i18n.i18n import I18nAuto -from configs.config import Config, GPU_INDEX, GPU_INFOS, GPU_MEMORY, IS_GPU import torch, platform import numpy as np import gradio as gr @@ -143,6 +143,7 @@ print( i18n("当前设备：%s | 推理精度：%s") % (config.device, config.dtype), flush=True, ) +logger.info("RVC_CUDA_GRAPH=%s", os.environ.get("RVC_CUDA_GRAPH", "0")) # GPU filtering and precision rules are shared with inference/extraction/training. gpu_infos = list(GPU_INFOS) gpu_indices = sorted(GPU_INDEX) @@ -394,6 +395,10 @@ def train_task_stopped(state): def start_train_process(state, cmd): kwargs = {"shell": True, "cwd": now_dir} + if "train/train.py" in cmd.replace("\\", "/"): + training_env = os.environ.copy() + training_env["RVC_CUDA_GRAPH"] = "0" + kwargs["env"] = training_env if platform.system() == "Windows": kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP else: