mirror of
https://github.com/serp-ai/bark-with-voice-clone.git
synced 2025-12-16 03:38:01 +01:00
Add RVC support
This commit is contained in:
Francis LaBounty
280
train_fine.ipynb
280
train_fine.ipynb
@@ -10,56 +10,9 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"WARNING:root:WARNING: Could not find module 'C:\\Users\\labou\\AppData\\Local\\Programs\\Python\\Python310\\Lib\\site-packages\\xformers\\_C.pyd' (or one of its dependencies). Try using the full path with constructor syntax.\n",
|
||||
"Need to compile C++ extensions to get sparse attention suport. Please run python setup.py build develop\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Could not find module 'C:\\Users\\labou\\AppData\\Local\\Programs\\Python\\Python310\\Lib\\site-packages\\xformers\\_C.pyd' (or one of its dependencies). Try using the full path with constructor syntax.\n",
|
||||
"\n",
|
||||
"===================================BUG REPORT===================================\n",
|
||||
"Welcome to bitsandbytes. For bug reports, please run\n",
|
||||
"\n",
|
||||
"python -m bitsandbytes\n",
|
||||
"\n",
|
||||
" and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
|
||||
"================================================================================\n",
|
||||
"bin c:\\Users\\labou\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\bitsandbytes\\libbitsandbytes_cuda116.dll\n",
|
||||
"function 'cadam32bit_grad_fp32' not found\n",
|
||||
"CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching in backup paths...\n",
|
||||
"CUDA SETUP: WARNING! libcuda.so not found! Do you have a CUDA driver installed? If you are on a cluster, make sure you are on a CUDA machine!\n",
|
||||
"CUDA SETUP: Loading binary c:\\Users\\labou\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\bitsandbytes\\libbitsandbytes_cuda116.dll...\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"c:\\Users\\labou\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\bitsandbytes\\cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable.\n",
|
||||
" warn(\"The installed version of bitsandbytes was compiled without GPU support. \"\n",
|
||||
"c:\\Users\\labou\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\bitsandbytes\\cuda_setup\\main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {WindowsPath('vs/workbench/api/node/extensionHostProcess')}\n",
|
||||
" warn(msg)\n",
|
||||
"c:\\Users\\labou\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\bitsandbytes\\cuda_setup\\main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {WindowsPath('module'), WindowsPath('/matplotlib_inline.backend_inline')}\n",
|
||||
" warn(msg)\n",
|
||||
"c:\\Users\\labou\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\bitsandbytes\\cuda_setup\\main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {WindowsPath('/usr/local/cuda/lib64')}\n",
|
||||
" warn(msg)\n",
|
||||
"c:\\Users\\labou\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\bitsandbytes\\cuda_setup\\main.py:149: UserWarning: WARNING: No libcudart.so found! Install CUDA or the cudatoolkit package (anaconda)!\n",
|
||||
" warn(msg)\n",
|
||||
"c:\\Users\\labou\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\bitsandbytes\\cuda_setup\\main.py:149: UserWarning: WARNING: No GPU detected! Check your CUDA paths. Proceeding to load CPU-only library...\n",
|
||||
" warn(msg)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"import torch.nn as nn\n",
|
||||
@@ -98,13 +51,13 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"train_batch_size = 8\n",
|
||||
"eval_batch_size = 8\n",
|
||||
"grad_accum = 1\n",
|
||||
"grad_accum = 2\n",
|
||||
"ckpt_path = 'models/fine_2.pt'\n",
|
||||
"model_type = \"fine\"\n",
|
||||
"dataset_path = 'datasets/joe_biden_state_of_union/'\n",
|
||||
@@ -128,7 +81,7 @@
|
||||
"lora_scaling = 1\n",
|
||||
"lora_dropout = 0.1\n",
|
||||
"lora_module_name = 'transformer.h'\n",
|
||||
"optimize_lora_params_only = True\n",
|
||||
"optimize_lora_params_only = False\n",
|
||||
"\n",
|
||||
"learning_rate = 1e-4\n",
|
||||
"scale_lr = False\n",
|
||||
@@ -142,13 +95,11 @@
|
||||
"keep_in_fp32_modules = ['lm_head']\n",
|
||||
"\n",
|
||||
"lr_scheduler_type = 'linear'\n",
|
||||
"lr_warmup_steps = 100\n",
|
||||
"lr_warmup_steps = 60\n",
|
||||
"num_train_epochs = 5\n",
|
||||
"max_train_steps = None\n",
|
||||
"max_grad_norm = 1.0\n",
|
||||
"\n",
|
||||
"semantic_cross_entropy_loss_weight = 0\n",
|
||||
"\n",
|
||||
"seed = 741"
|
||||
]
|
||||
},
|
||||
@@ -162,18 +113,9 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"c:\\Users\\labou\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\accelerate\\accelerator.py:258: FutureWarning: `logging_dir` is deprecated and will be removed in version 0.18.0 of 🤗 Accelerate. Use `project_dir` instead.\n",
|
||||
" warnings.warn(\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"CONTEXT_WINDOW_SIZE = 1024\n",
|
||||
"\n",
|
||||
@@ -342,7 +284,7 @@
|
||||
"\n",
|
||||
"\n",
|
||||
"def load_filepaths_and_text(filename, split=\"|\"):\n",
|
||||
" with open(filename, encoding='utf-8') as f:\n",
|
||||
" with open(filename, encoding='utf-8', errors='ignore') as f:\n",
|
||||
" filepaths_and_text = [line.strip().split(split) for line in f]\n",
|
||||
" base = os.path.dirname(filename)\n",
|
||||
" for j in range(len(filepaths_and_text)):\n",
|
||||
@@ -414,7 +356,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -529,24 +471,16 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Loaded fine model with 302090240 params, val_loss=2.0786.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model = _load_model(ckpt_path, device, use_small=False, model_type=model_type)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -570,7 +504,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -626,7 +560,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -659,7 +593,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -678,7 +612,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -726,7 +660,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -744,67 +678,9 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"ERROR:wandb.jupyter:Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n",
|
||||
"\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mfrancislabounty\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"wandb version 0.15.4 is available! To upgrade, please run:\n",
|
||||
" $ pip install wandb --upgrade"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"Tracking run with wandb version 0.13.6"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"Run data is saved locally in <code>e:\\Python\\bark-with-voice-clone\\wandb\\run-20230629_202416-290ebk11</code>"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"Syncing run <strong><a href=\"https://wandb.ai/francislabounty/bark_coarse/runs/290ebk11\" target=\"_blank\">fresh-pyramid-26</a></strong> to <a href=\"https://wandb.ai/francislabounty/bark_coarse\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://wandb.me/run\" target=\"_blank\">docs</a>)<br/>"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# We need to recalculate our total training steps as the size of the training dataloader may have changed.\n",
|
||||
"num_update_steps_per_epoch = math.ceil(len(train_dataloader) / grad_accum)\n",
|
||||
@@ -850,17 +726,9 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Validation Loss: 30.702054630626332 over 82 samples and 11 batches.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if accelerator.is_main_process:\n",
|
||||
" model.eval()\n",
|
||||
@@ -883,7 +751,7 @@
|
||||
" loss_7 = criterion(logits_7.view(-1, model.config.output_vocab_size), fine_targets_7.view(-1))\n",
|
||||
" loss_8 = criterion(logits_8.view(-1, model.config.output_vocab_size), fine_targets_8.view(-1))\n",
|
||||
"\n",
|
||||
" loss = loss_7 + loss_8\n",
|
||||
" loss = (loss_7 + loss_8) / 2\n",
|
||||
" validation_loss += loss.item()\n",
|
||||
" num_batches += 1\n",
|
||||
" num_samples += val_batch['fine_tokens'].size(0)\n",
|
||||
@@ -903,91 +771,9 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "3761107b0c094d2db6532410a582408c",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/205 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "45d15ef5bc1e4729aaf827aa3380823d",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"VBox(children=(Label(value='0.001 MB of 0.001 MB uploaded (0.000 MB deduped)\\r'), FloatProgress(value=1.0, max…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<style>\n",
|
||||
" table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
|
||||
" .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
|
||||
" .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
|
||||
" </style>\n",
|
||||
"<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>loss</td><td>█▆█▅▅▇▆▆▆▇▆▆▅▄▃▃▄▂▃▁▁▁▁▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁</td></tr><tr><td>lr</td><td>▁▁▂▂▂▃▃▄▄▄▅▅▅▆▆▆▇▇███▇▇▇▆▆▆▅▅▅▄▄▄▃▃▂▂▂▁▁</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>loss</td><td>3.18231</td></tr><tr><td>lr</td><td>0.0</td></tr></table><br/></div></div>"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"Synced <strong style=\"color:#cdcd00\">fresh-pyramid-26</strong>: <a href=\"https://wandb.ai/francislabounty/bark_coarse/runs/290ebk11\" target=\"_blank\">https://wandb.ai/francislabounty/bark_coarse/runs/290ebk11</a><br/>Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"Find logs at: <code>.\\wandb\\run-20230629_202416-290ebk11\\logs</code>"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Only show the progress bar once on each machine.\n",
|
||||
"progress_bar = tqdm(range(global_step, max_train_steps), disable=not accelerator.is_local_main_process)\n",
|
||||
@@ -1016,7 +802,7 @@
|
||||
" loss_7 = criterion(logits_7.view(-1, model.config.output_vocab_size), fine_targets_7.view(-1))\n",
|
||||
" loss_8 = criterion(logits_8.view(-1, model.config.output_vocab_size), fine_targets_8.view(-1))\n",
|
||||
"\n",
|
||||
" loss = loss_7 + loss_8\n",
|
||||
" loss = (loss_7 + loss_8) / 2\n",
|
||||
"\n",
|
||||
" accelerator.backward(loss)\n",
|
||||
" if accelerator.sync_gradients:\n",
|
||||
@@ -1072,17 +858,9 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Validation Loss: 3.041703635996038 over 82 samples and 11 batches.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if accelerator.is_main_process:\n",
|
||||
" model.eval()\n",
|
||||
@@ -1105,7 +883,7 @@
|
||||
" loss_7 = criterion(logits_7.view(-1, model.config.output_vocab_size), fine_targets_7.view(-1))\n",
|
||||
" loss_8 = criterion(logits_8.view(-1, model.config.output_vocab_size), fine_targets_8.view(-1))\n",
|
||||
"\n",
|
||||
" loss = loss_7 + loss_8\n",
|
||||
" loss = (loss_7 + loss_8) / 2\n",
|
||||
" validation_loss += loss.item()\n",
|
||||
" num_batches += 1\n",
|
||||
" num_samples += val_batch['fine_tokens'].size(0)\n",
|
||||
|
||||
Reference in New Issue
Block a user