From 18359f4179af89804be0151588738d8016b131dc Mon Sep 17 00:00:00 2001 From: kalomaze <66376113+kalomaze@users.noreply.github.com> Date: Sun, 23 Jul 2023 04:36:30 -0500 Subject: [PATCH 1/4] Implement automatic per-epoch TensorBoard logging --- infer-web.py | 24 ++++++++++++++++++++++-- train/utils.py | 18 ++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/infer-web.py b/infer-web.py index ff88efa..b3eac89 100644 --- a/infer-web.py +++ b/infer-web.py @@ -2,6 +2,7 @@ import os import shutil import sys import json # Mangio fork using json for preset saving +import math import signal @@ -942,6 +943,20 @@ def change_f0(if_f0_3, sr2, version19): # f0method8,pretrained_G14,pretrained_D else "", ) +global log_interval + +def set_log_interval(exp_dir, batch_size12): + log_interval = 1 + + folder_path = os.path.join(exp_dir, "1_16k_wavs") + + if os.path.exists(folder_path) and os.path.isdir(folder_path): + wav_files = [f for f in os.listdir(folder_path) if f.endswith('.wav')] + if wav_files: + sample_size = len(wav_files) + log_interval = math.ceil(sample_size / batch_size12) + + return log_interval # but3.click(click_train,[exp_dir1,sr2,if_f0_3,save_epoch10,total_epoch11,batch_size12,if_save_latest13,pretrained_G14,pretrained_D15,gpus16]) def click_train( @@ -969,6 +984,9 @@ def click_train( if version19 == "v1" else "%s/3_feature768" % (exp_dir) ) + + log_interval = set_log_interval(exp_dir, batch_size12) + if if_f0_3: f0_dir = "%s/2a_f0" % (exp_dir) f0nsf_dir = "%s/2b-f0nsf" % (exp_dir) @@ -1038,7 +1056,7 @@ def click_train( #### cmd = ( config.python_cmd - + " train_nsf_sim_cache_sid_load_pretrain.py -e %s -sr %s -f0 %s -bs %s -g %s -te %s -se %s %s %s -l %s -c %s -sw %s -v %s" + + " train_nsf_sim_cache_sid_load_pretrain.py -e %s -sr %s -f0 %s -bs %s -g %s -te %s -se %s %s %s -l %s -c %s -sw %s -v %s -li %s" % ( exp_dir1, sr2, @@ -1053,12 +1071,13 @@ def click_train( 1 if if_cache_gpu17 == i18n("是") else 0, 1 if if_save_every_weights18 == i18n("是") else 0, version19, + log_interval, ) ) else: cmd = ( config.python_cmd - + " train_nsf_sim_cache_sid_load_pretrain.py -e %s -sr %s -f0 %s -bs %s -te %s -se %s %s %s -l %s -c %s -sw %s -v %s" + + " train_nsf_sim_cache_sid_load_pretrain.py -e %s -sr %s -f0 %s -bs %s -te %s -se %s %s %s -l %s -c %s -sw %s -v %s -li %s" % ( exp_dir1, sr2, @@ -1072,6 +1091,7 @@ def click_train( 1 if if_cache_gpu17 == i18n("是") else 0, 1 if if_save_every_weights18 == i18n("是") else 0, version19, + log_interval, ) ) print(cmd) diff --git a/train/utils.py b/train/utils.py index 8884e43..14182db 100644 --- a/train/utils.py +++ b/train/utils.py @@ -352,6 +352,13 @@ def get_hparams(init=True): required=True, help="if caching the dataset in GPU memory, 1 or 0", ) + parser.add_argument( + "-li", + "--log_interval", + type=int, + required=True, + help="log interval" + ) args = parser.parse_args() name = args.experiment_dir @@ -391,6 +398,17 @@ def get_hparams(init=True): hparams.save_every_weights = args.save_every_weights hparams.if_cache_data_in_gpu = args.if_cache_data_in_gpu hparams.data.training_files = "%s/filelist.txt" % experiment_dir + + hparams.log_interval = args.log_interval + + # Update log_interval in the 'train' section of the config dictionary + config['train']['log_interval'] = args.log_interval + print("Updated log interval.") + + # Save the updated config back to the config_save_path + with open(config_save_path, 'w') as f: + json.dump(config, f, indent=4) + return hparams From 3c4f150ccf046fcf79a8462d98537e7d0d27897b Mon Sep 17 00:00:00 2001 From: kalomaze <66376113+kalomaze@users.noreply.github.com> Date: Sun, 23 Jul 2023 04:40:24 -0500 Subject: [PATCH 2/4] it printed twice, idk why, but it worked --- train/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/train/utils.py b/train/utils.py index 14182db..d1e0558 100644 --- a/train/utils.py +++ b/train/utils.py @@ -403,7 +403,6 @@ def get_hparams(init=True): # Update log_interval in the 'train' section of the config dictionary config['train']['log_interval'] = args.log_interval - print("Updated log interval.") # Save the updated config back to the config_save_path with open(config_save_path, 'w') as f: From 795dea48b56663ba345d50ab9332d65c1b5b2092 Mon Sep 17 00:00:00 2001 From: kalomaze Date: Sun, 23 Jul 2023 09:41:39 +0000 Subject: [PATCH 3/4] Apply Code Formatter Change --- infer-web.py | 11 +++++++---- train/utils.py | 10 +++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/infer-web.py b/infer-web.py index b3eac89..faa15ed 100644 --- a/infer-web.py +++ b/infer-web.py @@ -943,21 +943,24 @@ def change_f0(if_f0_3, sr2, version19): # f0method8,pretrained_G14,pretrained_D else "", ) + global log_interval + def set_log_interval(exp_dir, batch_size12): log_interval = 1 - + folder_path = os.path.join(exp_dir, "1_16k_wavs") - + if os.path.exists(folder_path) and os.path.isdir(folder_path): - wav_files = [f for f in os.listdir(folder_path) if f.endswith('.wav')] + wav_files = [f for f in os.listdir(folder_path) if f.endswith(".wav")] if wav_files: sample_size = len(wav_files) log_interval = math.ceil(sample_size / batch_size12) - + return log_interval + # but3.click(click_train,[exp_dir1,sr2,if_f0_3,save_epoch10,total_epoch11,batch_size12,if_save_latest13,pretrained_G14,pretrained_D15,gpus16]) def click_train( exp_dir1, diff --git a/train/utils.py b/train/utils.py index d1e0558..903214d 100644 --- a/train/utils.py +++ b/train/utils.py @@ -353,11 +353,7 @@ def get_hparams(init=True): help="if caching the dataset in GPU memory, 1 or 0", ) parser.add_argument( - "-li", - "--log_interval", - type=int, - required=True, - help="log interval" + "-li", "--log_interval", type=int, required=True, help="log interval" ) args = parser.parse_args() @@ -402,10 +398,10 @@ def get_hparams(init=True): hparams.log_interval = args.log_interval # Update log_interval in the 'train' section of the config dictionary - config['train']['log_interval'] = args.log_interval + config["train"]["log_interval"] = args.log_interval # Save the updated config back to the config_save_path - with open(config_save_path, 'w') as f: + with open(config_save_path, "w") as f: json.dump(config, f, indent=4) return hparams From c2f7718439549fa24462373723e2232d0559f704 Mon Sep 17 00:00:00 2001 From: kalomaze <66376113+kalomaze@users.noreply.github.com> Date: Sun, 23 Jul 2023 05:03:39 -0500 Subject: [PATCH 4/4] Make it properly append to train section of json --- train/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train/utils.py b/train/utils.py index d1e0558..3fcf392 100644 --- a/train/utils.py +++ b/train/utils.py @@ -399,7 +399,7 @@ def get_hparams(init=True): hparams.if_cache_data_in_gpu = args.if_cache_data_in_gpu hparams.data.training_files = "%s/filelist.txt" % experiment_dir - hparams.log_interval = args.log_interval + hparams.train.log_interval = args.log_interval # Update log_interval in the 'train' section of the config dictionary config['train']['log_interval'] = args.log_interval