mirror of
https://github.com/AIGC-Audio/AudioGPT.git
synced 2025-12-17 12:27:51 +01:00
147 lines
4.2 KiB
Python
147 lines
4.2 KiB
Python
import librosa
|
|
import numpy as np
|
|
from pycwt import wavelet
|
|
from scipy.interpolate import interp1d
|
|
|
|
|
|
def load_wav(wav_file, sr):
|
|
wav, _ = librosa.load(wav_file, sr=sr, mono=True)
|
|
return wav
|
|
|
|
|
|
def convert_continuos_f0(f0):
|
|
'''CONVERT F0 TO CONTINUOUS F0
|
|
Args:
|
|
f0 (ndarray): original f0 sequence with the shape (T)
|
|
Return:
|
|
(ndarray): continuous f0 with the shape (T)
|
|
'''
|
|
# get uv information as binary
|
|
f0 = np.copy(f0)
|
|
uv = np.float32(f0 != 0)
|
|
|
|
# get start and end of f0
|
|
if (f0 == 0).all():
|
|
print("| all of the f0 values are 0.")
|
|
return uv, f0
|
|
start_f0 = f0[f0 != 0][0]
|
|
end_f0 = f0[f0 != 0][-1]
|
|
|
|
# padding start and end of f0 sequence
|
|
start_idx = np.where(f0 == start_f0)[0][0]
|
|
end_idx = np.where(f0 == end_f0)[0][-1]
|
|
f0[:start_idx] = start_f0
|
|
f0[end_idx:] = end_f0
|
|
|
|
# get non-zero frame index
|
|
nz_frames = np.where(f0 != 0)[0]
|
|
|
|
# perform linear interpolation
|
|
f = interp1d(nz_frames, f0[nz_frames])
|
|
cont_f0 = f(np.arange(0, f0.shape[0]))
|
|
|
|
return uv, cont_f0
|
|
|
|
|
|
def get_cont_lf0(f0, frame_period=5.0):
|
|
uv, cont_f0_lpf = convert_continuos_f0(f0)
|
|
# cont_f0_lpf = low_pass_filter(cont_f0_lpf, int(1.0 / (frame_period * 0.001)), cutoff=20)
|
|
cont_lf0_lpf = np.log(cont_f0_lpf)
|
|
return uv, cont_lf0_lpf
|
|
|
|
|
|
def get_lf0_cwt(lf0):
|
|
'''
|
|
input:
|
|
signal of shape (N)
|
|
output:
|
|
Wavelet_lf0 of shape(10, N), scales of shape(10)
|
|
'''
|
|
mother = wavelet.MexicanHat()
|
|
dt = 0.005
|
|
dj = 1
|
|
s0 = dt * 2
|
|
J = 9
|
|
|
|
Wavelet_lf0, scales, _, _, _, _ = wavelet.cwt(np.squeeze(lf0), dt, dj, s0, J, mother)
|
|
# Wavelet.shape => (J + 1, len(lf0))
|
|
Wavelet_lf0 = np.real(Wavelet_lf0).T
|
|
return Wavelet_lf0, scales
|
|
|
|
|
|
def norm_scale(Wavelet_lf0):
|
|
Wavelet_lf0_norm = np.zeros((Wavelet_lf0.shape[0], Wavelet_lf0.shape[1]))
|
|
mean = Wavelet_lf0.mean(0)[None, :]
|
|
std = Wavelet_lf0.std(0)[None, :]
|
|
Wavelet_lf0_norm = (Wavelet_lf0 - mean) / std
|
|
return Wavelet_lf0_norm, mean, std
|
|
|
|
|
|
def normalize_cwt_lf0(f0, mean, std):
|
|
uv, cont_lf0_lpf = get_cont_lf0(f0)
|
|
cont_lf0_norm = (cont_lf0_lpf - mean) / std
|
|
Wavelet_lf0, scales = get_lf0_cwt(cont_lf0_norm)
|
|
Wavelet_lf0_norm, _, _ = norm_scale(Wavelet_lf0)
|
|
|
|
return Wavelet_lf0_norm
|
|
|
|
|
|
def get_lf0_cwt_norm(f0s, mean, std):
|
|
uvs = list()
|
|
cont_lf0_lpfs = list()
|
|
cont_lf0_lpf_norms = list()
|
|
Wavelet_lf0s = list()
|
|
Wavelet_lf0s_norm = list()
|
|
scaless = list()
|
|
|
|
means = list()
|
|
stds = list()
|
|
for f0 in f0s:
|
|
uv, cont_lf0_lpf = get_cont_lf0(f0)
|
|
cont_lf0_lpf_norm = (cont_lf0_lpf - mean) / std
|
|
|
|
Wavelet_lf0, scales = get_lf0_cwt(cont_lf0_lpf_norm) # [560,10]
|
|
Wavelet_lf0_norm, mean_scale, std_scale = norm_scale(Wavelet_lf0) # [560,10],[1,10],[1,10]
|
|
|
|
Wavelet_lf0s_norm.append(Wavelet_lf0_norm)
|
|
uvs.append(uv)
|
|
cont_lf0_lpfs.append(cont_lf0_lpf)
|
|
cont_lf0_lpf_norms.append(cont_lf0_lpf_norm)
|
|
Wavelet_lf0s.append(Wavelet_lf0)
|
|
scaless.append(scales)
|
|
means.append(mean_scale)
|
|
stds.append(std_scale)
|
|
|
|
return Wavelet_lf0s_norm, scaless, means, stds
|
|
|
|
|
|
def inverse_cwt_torch(Wavelet_lf0, scales):
|
|
import torch
|
|
b = ((torch.arange(0, len(scales)).float().to(Wavelet_lf0.device)[None, None, :] + 1 + 2.5) ** (-2.5))
|
|
lf0_rec = Wavelet_lf0 * b
|
|
lf0_rec_sum = lf0_rec.sum(-1)
|
|
lf0_rec_sum = (lf0_rec_sum - lf0_rec_sum.mean(-1, keepdim=True)) / lf0_rec_sum.std(-1, keepdim=True)
|
|
return lf0_rec_sum
|
|
|
|
|
|
def inverse_cwt(Wavelet_lf0, scales):
|
|
b = ((np.arange(0, len(scales))[None, None, :] + 1 + 2.5) ** (-2.5))
|
|
lf0_rec = Wavelet_lf0 * b
|
|
lf0_rec_sum = lf0_rec.sum(-1)
|
|
lf0_rec_sum = (lf0_rec_sum - lf0_rec_sum.mean(-1, keepdims=True)) / lf0_rec_sum.std(-1, keepdims=True)
|
|
return lf0_rec_sum
|
|
|
|
|
|
def cwt2f0(cwt_spec, mean, std, cwt_scales):
|
|
assert len(mean.shape) == 1 and len(std.shape) == 1 and len(cwt_spec.shape) == 3
|
|
import torch
|
|
if isinstance(cwt_spec, torch.Tensor):
|
|
f0 = inverse_cwt_torch(cwt_spec, cwt_scales)
|
|
f0 = f0 * std[:, None] + mean[:, None]
|
|
f0 = f0.exp() # [B, T]
|
|
else:
|
|
f0 = inverse_cwt(cwt_spec, cwt_scales)
|
|
f0 = f0 * std[:, None] + mean[:, None]
|
|
f0 = np.exp(f0) # [B, T]
|
|
return f0
|