Mirrors/Retrieval-based-Voice-Conversion-WebUI
1
0
Fork 0
mirror of https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI.git synced 2026-08-29 10:09:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

Use GPU processing for UVR5 and input audio loading and resampling where possible to improve inference efficiency and reduce CPU usage

Browse Source
This commit is contained in:
RVC-Boss
2026-07-21 21:23:25 +08:00
parent 6ce75e3a4a
commit 54465d6519
26 changed files with 4666 additions and 4126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

324
tools/uvr5/vr.py
View File

@@ -5,10 +5,16 @@ import logging
logger = logging.getLogger(__name__)
import librosa
import numpy as np
import soundfile as sf
import torch
from infer.audio import (
TORCHAUDIO_GPU_ENABLED,
load_audio,
load_audio_tensor,
resample_audio,
resample_audio_tensor,
)
from tools.uvr5.lib.lib_v5 import nets_61968KB as Nets
from tools.uvr5.lib.lib_v5 import spec_utils
from tools.uvr5.lib.lib_v5.model_param_init import ModelParameters
@@ -16,6 +22,152 @@ from tools.uvr5.lib.lib_v5.nets_new import CascadedNet
from tools.uvr5.lib.utils import inference
def _ensure_stereo(audio):
audio = np.asarray(audio, dtype=np.float32)
if audio.ndim == 1:
audio = audio[np.newaxis, :]
if audio.shape[0] == 1:
return np.repeat(audio, 2, axis=0)
if audio.shape[0] > 2:
return np.ascontiguousarray(audio[:2])
return audio
def _ensure_stereo_tensor(audio, device):
if audio.ndim == 1:
audio = audio.unsqueeze(0)
if audio.shape[0] == 1:
audio = audio.repeat(2, 1)
elif audio.shape[0] > 2:
audio = audio[:2]
return audio.to(device=device)
def _cuda_device(device):
parsed = device if isinstance(device, torch.device) else torch.device(device)
return parsed if parsed.type == "cuda" else None
def _vr_gpu_memory_fits(audio, mp, device):
highest_band = len(mp.param["band"])
frames = max(
1,
int(audio.shape[-1] // mp.param["band"][highest_band]["hl"] + 1),
)
band_bins = sum(
mp.param["band"][band]["n_fft"] // 2 + 1
for band in mp.param["band"]
)
combined_bins = mp.param["bins"] + 1
# Complex band spectra + combined/target spectra + magnitude/prediction.
estimated = frames * 2 * (
band_bins * 8 + combined_bins * (8 * 3 + 4 * 3)
)
free_bytes, _ = torch.cuda.mem_get_info(device)
return estimated <= int(free_bytes * 0.42)
def _prepare_spectrogram(music_file, mp, data, device, allow_gpu=True):
cuda_device = _cuda_device(device)
use_gpu = bool(
allow_gpu and cuda_device is not None and TORCHAUDIO_GPU_ENABLED
)
if use_gpu:
try:
high_sr = mp.param["band"][len(mp.param["band"])]["sr"]
high_wave = _ensure_stereo_tensor(
load_audio_tensor(music_file, high_sr, force_mono=False),
cuda_device,
)
if not _vr_gpu_memory_fits(high_wave, mp, cuda_device):
use_gpu = False
high_wave = high_wave.float().cpu().numpy()
except torch.cuda.OutOfMemoryError:
torch.cuda.empty_cache()
use_gpu = False
high_wave = None
else:
high_wave = None
input_high_end_h = None
input_high_end = None
X_spec_s = {}
bands_n = len(mp.param["band"])
previous_wave = None
for d in range(bands_n, 0, -1):
bp = mp.param["band"][d]
if d == bands_n:
if high_wave is None:
current_wave = _ensure_stereo(
load_audio(music_file, bp["sr"], force_mono=False)
)
else:
current_wave = high_wave
elif use_gpu:
current_wave = resample_audio_tensor(
previous_wave,
mp.param["band"][d + 1]["sr"],
bp["sr"],
force_mono=False,
)
else:
current_wave = resample_audio(
previous_wave,
mp.param["band"][d + 1]["sr"],
bp["sr"],
force_mono=False,
res_type=bp["res_type"],
)
X_spec_s[d] = spec_utils.wave_to_spectrogram_mt(
current_wave,
bp["hl"],
bp["n_fft"],
mp.param["mid_side"],
mp.param["mid_side_b2"],
mp.param["reverse"],
)
if d == bands_n and data["high_end_process"] != "none":
input_high_end_h = (bp["n_fft"] // 2 - bp["crop_stop"]) + (
mp.param["pre_filter_stop"] - mp.param["pre_filter_start"]
)
input_high_end = X_spec_s[d][
:, bp["n_fft"] // 2 - input_high_end_h : bp["n_fft"] // 2, :
]
if torch.is_tensor(input_high_end):
input_high_end = input_high_end.clone()
previous_wave = current_wave
X_spec_m = spec_utils.combine_spectrograms(X_spec_s, mp)
del previous_wave, X_spec_s
return X_spec_m, input_high_end_h, input_high_end
def _wave_for_write(wave):
if torch.is_tensor(wave):
return wave.detach().to(device="cpu", dtype=torch.float32).numpy()
return np.asarray(wave)
def _separate_spectrogram(X_spec_m, device, model, aggressiveness, data):
with torch.no_grad():
pred, X_mag, X_phase = inference(
X_spec_m, device, model, aggressiveness, data
)
if data["postprocess"]:
if torch.is_tensor(pred):
pred_inv = torch.clamp(X_mag - pred, min=0)
else:
pred_inv = np.clip(X_mag - pred, 0, np.inf)
pred = spec_utils.mask_silence(pred, pred_inv)
if torch.is_tensor(X_spec_m):
ratio = pred.float() / X_mag.clamp_min(1e-8)
ratio = torch.nan_to_num(ratio)
y_spec_m = X_spec_m * ratio
else:
y_spec_m = pred * X_phase
return y_spec_m
class AudioPre:
def __init__(self, agg, model_path, device, is_half, tta=False):
self.model_path = model_path
@@ -50,61 +202,32 @@ class AudioPre:
os.makedirs(ins_root, exist_ok=True)
if vocal_root is not None:
os.makedirs(vocal_root, exist_ok=True)
X_wave, y_wave, X_spec_s, y_spec_s = {}, {}, {}, {}
bands_n = len(self.mp.param["band"])
# print(bands_n)
for d in range(bands_n, 0, -1):
bp = self.mp.param["band"][d]
if d == bands_n: # high-end band
(
X_wave[d],
_,
) = librosa.core.load( # 理论上librosa读取可能对某些音频有bug，应该上ffmpeg读取，但是太麻烦了弃坑
music_file,
sr=bp["sr"],
mono=False,
dtype=np.float32,
res_type=bp["res_type"],
)
if X_wave[d].ndim == 1:
X_wave[d] = np.asfortranarray([X_wave[d], X_wave[d]])
else: # lower bands
X_wave[d] = librosa.core.resample(
X_wave[d + 1],
orig_sr=self.mp.param["band"][d + 1]["sr"],
target_sr=bp["sr"],
res_type=bp["res_type"],
)
# Stft of wave source
X_spec_s[d] = spec_utils.wave_to_spectrogram_mt(
X_wave[d],
bp["hl"],
bp["n_fft"],
self.mp.param["mid_side"],
self.mp.param["mid_side_b2"],
self.mp.param["reverse"],
)
# pdb.set_trace()
if d == bands_n and self.data["high_end_process"] != "none":
input_high_end_h = (bp["n_fft"] // 2 - bp["crop_stop"]) + (
self.mp.param["pre_filter_stop"] - self.mp.param["pre_filter_start"]
)
input_high_end = X_spec_s[d][:, bp["n_fft"] // 2 - input_high_end_h : bp["n_fft"] // 2, :]
X_spec_m = spec_utils.combine_spectrograms(X_spec_s, self.mp)
aggresive_set = float(self.data["agg"] / 100)
aggressiveness = {
"value": aggresive_set,
"split_bin": self.mp.param["band"][1]["crop_stop"],
}
with torch.no_grad():
pred, X_mag, X_phase = inference(X_spec_m, self.device, self.model, aggressiveness, self.data)
# Postprocess
if self.data["postprocess"]:
pred_inv = np.clip(X_mag - pred, 0, np.inf)
pred = spec_utils.mask_silence(pred, pred_inv)
y_spec_m = pred * X_phase
v_spec_m = X_spec_m - y_spec_m
gpu_oom = False
try:
X_spec_m, input_high_end_h, input_high_end = _prepare_spectrogram(
music_file, self.mp, self.data, self.device
)
y_spec_m = _separate_spectrogram(
X_spec_m, self.device, self.model, aggressiveness, self.data
)
except torch.cuda.OutOfMemoryError:
X_spec_m = None
input_high_end = None
y_spec_m = None
gpu_oom = True
if gpu_oom:
torch.cuda.empty_cache()
X_spec_m, input_high_end_h, input_high_end = _prepare_spectrogram(
music_file, self.mp, self.data, self.device, allow_gpu=False
)
y_spec_m = _separate_spectrogram(
X_spec_m, self.device, self.model, aggressiveness, self.data
)
if is_hp3 == True:
ins_root, vocal_root = vocal_root, ins_root
@@ -128,14 +251,14 @@ class AudioPre:
ins_root,
head + "{}_{}.{}".format(name, self.data["agg"], format),
),
(np.array(wav_instrument) * 32768).astype("int16"),
(_wave_for_write(wav_instrument) * 32768).astype("int16"),
self.mp.param["sr"],
) #
else:
path = os.path.join(ins_root, head + "{}_{}.wav".format(name, self.data["agg"]))
sf.write(
path,
(np.array(wav_instrument) * 32768).astype("int16"),
(_wave_for_write(wav_instrument) * 32768).astype("int16"),
self.mp.param["sr"],
)
if os.path.exists(path):
@@ -149,6 +272,12 @@ class AudioPre:
except:
pass
if vocal_root is not None:
if torch.is_tensor(y_spec_m):
y_spec_m.neg_().add_(X_spec_m)
v_spec_m = y_spec_m
else:
np.subtract(X_spec_m, y_spec_m, out=y_spec_m)
v_spec_m = y_spec_m
if is_hp3 == True:
head = "instrument_"
else:
@@ -165,14 +294,14 @@ class AudioPre:
vocal_root,
head + "{}_{}.{}".format(name, self.data["agg"], format),
),
(np.array(wav_vocals) * 32768).astype("int16"),
(_wave_for_write(wav_vocals) * 32768).astype("int16"),
self.mp.param["sr"],
)
else:
path = os.path.join(vocal_root, head + "{}_{}.wav".format(name, self.data["agg"]))
sf.write(
path,
(np.array(wav_vocals) * 32768).astype("int16"),
(_wave_for_write(wav_vocals) * 32768).astype("int16"),
self.mp.param["sr"],
)
if os.path.exists(path):
@@ -224,61 +353,32 @@ class AudioPreDeEcho:
os.makedirs(ins_root, exist_ok=True)
if vocal_root is not None:
os.makedirs(vocal_root, exist_ok=True)
X_wave, y_wave, X_spec_s, y_spec_s = {}, {}, {}, {}
bands_n = len(self.mp.param["band"])
# print(bands_n)
for d in range(bands_n, 0, -1):
bp = self.mp.param["band"][d]
if d == bands_n: # high-end band
(
X_wave[d],
_,
) = librosa.core.load( # 理论上librosa读取可能对某些音频有bug，应该上ffmpeg读取，但是太麻烦了弃坑
music_file,
sr=bp["sr"],
mono=False,
dtype=np.float32,
res_type=bp["res_type"],
)
if X_wave[d].ndim == 1:
X_wave[d] = np.asfortranarray([X_wave[d], X_wave[d]])
else: # lower bands
X_wave[d] = librosa.core.resample(
X_wave[d + 1],
orig_sr=self.mp.param["band"][d + 1]["sr"],
target_sr=bp["sr"],
res_type=bp["res_type"],
)
# Stft of wave source
X_spec_s[d] = spec_utils.wave_to_spectrogram_mt(
X_wave[d],
bp["hl"],
bp["n_fft"],
self.mp.param["mid_side"],
self.mp.param["mid_side_b2"],
self.mp.param["reverse"],
)
# pdb.set_trace()
if d == bands_n and self.data["high_end_process"] != "none":
input_high_end_h = (bp["n_fft"] // 2 - bp["crop_stop"]) + (
self.mp.param["pre_filter_stop"] - self.mp.param["pre_filter_start"]
)
input_high_end = X_spec_s[d][:, bp["n_fft"] // 2 - input_high_end_h : bp["n_fft"] // 2, :]
X_spec_m = spec_utils.combine_spectrograms(X_spec_s, self.mp)
aggresive_set = float(self.data["agg"] / 100)
aggressiveness = {
"value": aggresive_set,
"split_bin": self.mp.param["band"][1]["crop_stop"],
}
with torch.no_grad():
pred, X_mag, X_phase = inference(X_spec_m, self.device, self.model, aggressiveness, self.data)
# Postprocess
if self.data["postprocess"]:
pred_inv = np.clip(X_mag - pred, 0, np.inf)
pred = spec_utils.mask_silence(pred, pred_inv)
y_spec_m = pred * X_phase
v_spec_m = X_spec_m - y_spec_m
gpu_oom = False
try:
X_spec_m, input_high_end_h, input_high_end = _prepare_spectrogram(
music_file, self.mp, self.data, self.device
)
y_spec_m = _separate_spectrogram(
X_spec_m, self.device, self.model, aggressiveness, self.data
)
except torch.cuda.OutOfMemoryError:
X_spec_m = None
input_high_end = None
y_spec_m = None
gpu_oom = True
if gpu_oom:
torch.cuda.empty_cache()
X_spec_m, input_high_end_h, input_high_end = _prepare_spectrogram(
music_file, self.mp, self.data, self.device, allow_gpu=False
)
y_spec_m = _separate_spectrogram(
X_spec_m, self.device, self.model, aggressiveness, self.data
)
if ins_root is not None:
if self.data["high_end_process"].startswith("mirroring"):
@@ -295,14 +395,14 @@ class AudioPreDeEcho:
ins_root,
"vocal_{}_{}.{}".format(name, self.data["agg"], format),
),
(np.array(wav_instrument) * 32768).astype("int16"),
(_wave_for_write(wav_instrument) * 32768).astype("int16"),
self.mp.param["sr"],
) #
else:
path = os.path.join(ins_root, "vocal_{}_{}.wav".format(name, self.data["agg"]))
sf.write(
path,
(np.array(wav_instrument) * 32768).astype("int16"),
(_wave_for_write(wav_instrument) * 32768).astype("int16"),
self.mp.param["sr"],
)
if os.path.exists(path):
@@ -316,6 +416,12 @@ class AudioPreDeEcho:
except:
pass
if vocal_root is not None:
if torch.is_tensor(y_spec_m):
y_spec_m.neg_().add_(X_spec_m)
v_spec_m = y_spec_m
else:
np.subtract(X_spec_m, y_spec_m, out=y_spec_m)
v_spec_m = y_spec_m
if self.data["high_end_process"].startswith("mirroring"):
input_high_end_ = spec_utils.mirroring(self.data["high_end_process"], v_spec_m, input_high_end, self.mp)
wav_vocals = spec_utils.cmb_spectrogram_to_wave(v_spec_m, self.mp, input_high_end_h, input_high_end_)
@@ -328,14 +434,14 @@ class AudioPreDeEcho:
vocal_root,
"instrument_{}_{}.{}".format(name, self.data["agg"], format),
),
(np.array(wav_vocals) * 32768).astype("int16"),
(_wave_for_write(wav_vocals) * 32768).astype("int16"),
self.mp.param["sr"],
)
else:
path = os.path.join(vocal_root, "instrument_{}_{}.wav".format(name, self.data["agg"]))
sf.write(
path,
(np.array(wav_vocals) * 32768).astype("int16"),
(_wave_for_write(wav_vocals) * 32768).astype("int16"),
self.mp.param["sr"],
)
if os.path.exists(path):