From 61a1d59ac5fe693320360cb7bf913d87e70a87bf Mon Sep 17 00:00:00 2001 From: PNRxA Date: Sat, 25 Apr 2020 16:30:19 +1000 Subject: [PATCH 1/8] numpy to use CPU when using CUDA --- utils/generic_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/generic_utils.py b/utils/generic_utils.py index 5d91d74d..435d2b10 100644 --- a/utils/generic_utils.py +++ b/utils/generic_utils.py @@ -157,7 +157,7 @@ def check_update(model, grad_clip, ignore_stopnet=False): grad_norm = torch.nn.utils.clip_grad_norm_([param for name, param in model.named_parameters() if 'stopnet' not in name], grad_clip) else: grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), grad_clip) - if np.isinf(grad_norm): + if np.isinf(grad_norm.cpu()): print(" | > Gradient is INF !!") skip_flag = True return grad_norm, skip_flag From e4e29f716e5bc2049d3316896401fae64933a716 Mon Sep 17 00:00:00 2001 From: Edresson Casanova Date: Mon, 4 May 2020 17:39:35 -0300 Subject: [PATCH 2/8] fix bug in bidirectional decoder train --- train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.py b/train.py index 94ccfedb..1444e103 100644 --- a/train.py +++ b/train.py @@ -356,7 +356,7 @@ def evaluate(model, criterion, ap, global_step, epoch): mel_lengths, decoder_backward_output, alignments, alignment_lengths, text_lengths) if c.bidirectional_decoder: - keep_avg.update_values({'avg_decoder_b_loss': loss_dict['decoder_backward_loss'].item(), + keep_avg.update_values({'avg_decoder_b_loss': loss_dict['decoder_b_loss'].item(), 'avg_decoder_c_loss': loss_dict['decoder_c_loss'].item()}) if c.ga_alpha > 0: keep_avg.update_values({'avg_ga_loss': loss_dict['ga_loss'].item()}) From cce13ee245fab03717d2afa1860e35986ebd9de9 Mon Sep 17 00:00:00 2001 From: Edresson Casanova Date: Mon, 4 May 2020 17:52:58 -0300 Subject: [PATCH 3/8] Fix bug in Graves Attn On my machine at Graves attention the variable self.J ( self.J = torch.arange(0, inputs.shape[1]+2).to(inputs.device) + 0.5) is a LongTensor, but it must be a float tensor. So I get the following error: Traceback (most recent call last): File "train.py", line 704, in main(args) File "train.py", line 619, in main global_step, epoch) File "train.py", line 170, in train text_input, text_lengths, mel_input, speaker_embeddings=speaker_embeddings) File "/home/edresson/anaconda3/envs/TTS2/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in __call__ result = self.forward(*input, **kwargs) File "/mnt/edresson/DD/TTS/voice-clonning/TTS/tts_namespace/TTS/models/tacotron.py", line 121, in forward self.speaker_embeddings_projected) File "/home/edresson/anaconda3/envs/TTS2/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in __call__ result = self.forward(*input, **kwargs) File "/mnt/edresson/DD/TTS/voice-clonning/TTS/tts_namespace/TTS/layers/tacotron.py", line 435, in forward output, stop_token, attention = self.decode(inputs, mask) File "/mnt/edresson/DD/TTS/voice-clonning/TTS/tts_namespace/TTS/layers/tacotron.py", line 367, in decode self.attention_rnn_hidden, inputs, self.processed_inputs, mask) File "/home/edresson/anaconda3/envs/TTS2/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in __call__ result = self.forward(*input, **kwargs) File "/mnt/edresson/DD/TTS/voice-clonning/TTS/tts_namespace/TTS/layers/common_layers.py", line 180, in forward phi_t = g_t.unsqueeze(-1) * (1.0 / (1.0 + torch.sigmoid((mu_t.unsqueeze(-1) - j) / sig_t.unsqueeze(-1)))) RuntimeError: expected type torch.cuda.FloatTensor but got torch.cuda.LongTensor In addition the + 0.5 operation is canceled if it is a LongTensor. Test: >>> torch.arange(0, 10) tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) >>> torch.arange(0, 10) + 0.5 tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) >>> torch.arange(0, 10.0) + 0.5 tensor([0.5000, 1.5000, 2.5000, 3.5000, 4.5000, 5.5000, 6.5000, 7.5000, 8.5000, 9.5000]) To resolve this I forced the arrange range to float: self.J = torch.arange(0, inputs.shape[1]+2.0).to(inputs.device) + 0.5 --- layers/common_layers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/layers/common_layers.py b/layers/common_layers.py index 8b7ed125..78fa8b1c 100644 --- a/layers/common_layers.py +++ b/layers/common_layers.py @@ -138,7 +138,7 @@ class GravesAttention(nn.Module): def init_states(self, inputs): if self.J is None or inputs.shape[1]+1 > self.J.shape[-1]: - self.J = torch.arange(0, inputs.shape[1]+2).to(inputs.device) + 0.5 + self.J = torch.arange(0, inputs.shape[1]+2.0).to(inputs.device) + 0.5 self.attention_weights = torch.zeros(inputs.shape[0], inputs.shape[1]).to(inputs.device) self.mu_prev = torch.zeros(inputs.shape[0], self.K).to(inputs.device) From 85a822e3190f6c7207c28f7485ecb0ebe7e480ba Mon Sep 17 00:00:00 2001 From: mittimithai Date: Tue, 12 May 2020 15:02:24 -0700 Subject: [PATCH 4/8] small change for multispeaker just threads speaker_id through decoder.run_model --- server/synthesizer.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/synthesizer.py b/server/synthesizer.py index e9205bf1..411be928 100644 --- a/server/synthesizer.py +++ b/server/synthesizer.py @@ -164,16 +164,21 @@ class Synthesizer(object): sentences = list(filter(None, [s.strip() for s in sentences])) # remove empty sentences return sentences - def tts(self, text): + def tts(self, text, speaker_id=None): wavs = [] sens = self.split_into_sentences(text) print(sens) + + speaker_id = id_to_torch(speaker_id) ++ if speaker_id is not None and self.use_cuda: ++ speaker_id = speaker_id.cuda() + for sen in sens: # preprocess the given text inputs = text_to_seqvec(sen, self.tts_config, self.use_cuda) # synthesize voice decoder_output, postnet_output, alignments, _ = run_model( - self.tts_model, inputs, self.tts_config, False, None, None) + self.tts_model, inputs, self.tts_config, False, speaker_id, None) # convert outputs to numpy postnet_output, decoder_output, _ = parse_outputs( postnet_output, decoder_output, alignments) From a4aca623c34243e065bd49ee5520fd714e461c1a Mon Sep 17 00:00:00 2001 From: mittimithai Date: Tue, 12 May 2020 15:23:45 -0700 Subject: [PATCH 5/8] removed + chars silly mistake copy pasting --- server/synthesizer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/synthesizer.py b/server/synthesizer.py index 411be928..dee2cd4b 100644 --- a/server/synthesizer.py +++ b/server/synthesizer.py @@ -170,8 +170,8 @@ class Synthesizer(object): print(sens) speaker_id = id_to_torch(speaker_id) -+ if speaker_id is not None and self.use_cuda: -+ speaker_id = speaker_id.cuda() + if speaker_id is not None and self.use_cuda: + speaker_id = speaker_id.cuda() for sen in sens: # preprocess the given text From 42ff83f9b9e1151ccaba6d7f9d8f520d274a0d31 Mon Sep 17 00:00:00 2001 From: mittimithai Date: Tue, 12 May 2020 18:50:58 -0700 Subject: [PATCH 6/8] trying to fix trailing whitespace --- server/synthesizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/synthesizer.py b/server/synthesizer.py index dee2cd4b..3035a287 100644 --- a/server/synthesizer.py +++ b/server/synthesizer.py @@ -172,7 +172,7 @@ class Synthesizer(object): speaker_id = id_to_torch(speaker_id) if speaker_id is not None and self.use_cuda: speaker_id = speaker_id.cuda() - + for sen in sens: # preprocess the given text inputs = text_to_seqvec(sen, self.tts_config, self.use_cuda) From 25f466f2994dad707f0aa00db36187d6a43844ee Mon Sep 17 00:00:00 2001 From: mittimithai Date: Tue, 12 May 2020 19:02:37 -0700 Subject: [PATCH 7/8] more whitespace problems --- server/synthesizer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/server/synthesizer.py b/server/synthesizer.py index 3035a287..453e5827 100644 --- a/server/synthesizer.py +++ b/server/synthesizer.py @@ -168,7 +168,6 @@ class Synthesizer(object): wavs = [] sens = self.split_into_sentences(text) print(sens) - speaker_id = id_to_torch(speaker_id) if speaker_id is not None and self.use_cuda: speaker_id = speaker_id.cuda() From 65b9c7d3d655e34741ff5adf75bc1b2a6df22158 Mon Sep 17 00:00:00 2001 From: thllwg Date: Fri, 15 May 2020 10:19:52 +0200 Subject: [PATCH 8/8] number of workers as config parameter --- speaker_encoder/config.json | 1 + speaker_encoder/train.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/speaker_encoder/config.json b/speaker_encoder/config.json index 79c42bc0..0d0f8f68 100644 --- a/speaker_encoder/config.json +++ b/speaker_encoder/config.json @@ -34,6 +34,7 @@ "save_step": 1000, // Number of training steps expected to save traning stats and checkpoints. "print_step": 1, // Number of steps to log traning on console. "output_path": "/media/erogol/data_ssd/Models/libri_tts/speaker_encoder/", // DATASET-RELATED: output path for all training outputs. + "num_loader_workers": 0, // number of training data loader processes. Don't set it too big. 4-8 are good values. "model": { "input_dim": 40, "proj_dim": 128, diff --git a/speaker_encoder/train.py b/speaker_encoder/train.py index 19067401..0a137360 100644 --- a/speaker_encoder/train.py +++ b/speaker_encoder/train.py @@ -44,7 +44,7 @@ def setup_loader(ap, is_val=False, verbose=False): loader = DataLoader(dataset, batch_size=c.num_speakers_in_batch, shuffle=False, - num_workers=0, + num_workers=c.num_loader_workers, collate_fn=dataset.collate_fn) return loader