From a50342ed5d50caf959d054894bfa0334dda17d4b Mon Sep 17 00:00:00 2001
From: Wang Qiang <37444407+XDUWQ@users.noreply.github.com>
Date: Mon, 3 Jul 2023 13:52:35 +0800
Subject: [PATCH] Add stable diffusion tutorial ipynb (#346)

---
 .../finetune_stable_diffusion_dreambooth.py   |  2 +
 .../lora/finetune_stable_diffusion_lora.py    |  2 +
 .../pytorch/stable_diffusion/tutorial.ipynb   | 83 +++++++++++++++++++
 3 files changed, 87 insertions(+)
 create mode 100644 examples/pytorch/stable_diffusion/tutorial.ipynb

diff --git a/examples/pytorch/stable_diffusion/dreambooth/finetune_stable_diffusion_dreambooth.py b/examples/pytorch/stable_diffusion/dreambooth/finetune_stable_diffusion_dreambooth.py
index 1f38cff7..f852d752 100644
--- a/examples/pytorch/stable_diffusion/dreambooth/finetune_stable_diffusion_dreambooth.py
+++ b/examples/pytorch/stable_diffusion/dreambooth/finetune_stable_diffusion_dreambooth.py
@@ -113,4 +113,6 @@ pipe = pipeline(
     model_revision=args.model_revision)
 
 output = pipe({'text': args.prompt})
+# visualize the result on ipynb and save it
+output
 cv2.imwrite('./dreambooth_result.png', output['output_imgs'][0])
diff --git a/examples/pytorch/stable_diffusion/lora/finetune_stable_diffusion_lora.py b/examples/pytorch/stable_diffusion/lora/finetune_stable_diffusion_lora.py
index 183e817d..2561a6a7 100644
--- a/examples/pytorch/stable_diffusion/lora/finetune_stable_diffusion_lora.py
+++ b/examples/pytorch/stable_diffusion/lora/finetune_stable_diffusion_lora.py
@@ -66,4 +66,6 @@ pipe = pipeline(
     model_revision=args.model_revision)
 
 output = pipe({'text': args.prompt})
+# visualize the result on ipynb and save it
+output
 cv2.imwrite('./lora_result.png', output['output_imgs'][0])
diff --git a/examples/pytorch/stable_diffusion/tutorial.ipynb b/examples/pytorch/stable_diffusion/tutorial.ipynb
new file mode 100644
index 00000000..941b4e76
--- /dev/null
+++ b/examples/pytorch/stable_diffusion/tutorial.ipynb
@@ -0,0 +1,83 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Modelscope微调Stable Diffusion教程\n",
+    "## 原理讲解\n",
+    "\n",
+    "从头训练stable diffusion需要数十万美元和一个月以上的时间，巨额的算力和时间成本让普通人难以承受。因此最理想的手段是利用开源的stable diffusion模型，通过微调手段定制化属于自己的模型。近年涌现出很多有效的微调stable diffusion手段，如[Textual Inversion](https://arxiv.org/abs/2208.01618)、[Dreambooth](https://arxiv.org/pdf/2208.12242.pdf)、[Lora](https://arxiv.org/abs/2106.09685)、[Custom Diffusion](https://arxiv.org/pdf/2302.05543.pdf)等，Modelscope目前已经支持了Dreambooth和Lora两种方法。\n",
+    "\n",
+    "### Dreambooth\n",
+    "如果我们直接使用几张图片微调Stable Diffusion模型，很容易陷入“过拟合”的状态，通常的表现为模型生成的结果同质化且损失了泛化能力。除此之外，还容易遇到语言漂移的问题，严重影响了模型性能。Dreambooth提出了重建损失和特定类别先验保留损失相结合的方法来解决这一问题。\n",
+    "\n",
+    "### Lora\n",
+    "Lora的全称是Low-Rank Adaptation，是一种低阶自适应技术。这项技术起源于微调大型语言模型，在stable diffusion上也能取得非常好的效果。因为大模型是一般是过参数化的，它们有更小的内在维度，Lora模型主要依赖于这个低的内在维度去做任务适配。通过低秩分解(先降维再升维)来模拟参数的改变量，从而以极小的参数量来实现大模型的间接训练。\n",
+    "\n",
+    "如下图所示，Lora在原先的模型层中并行插入了可训练的排序分解矩阵层，这个矩阵层是由一个降维矩阵A和一个升维矩阵B组成的。降维矩阵A采用高斯分布初始化，升维矩阵B初始化为全0，保证训练开始时旁路为0矩阵。在训练的时候原模型固定，只训练降维矩阵A和升维矩阵B；在推理的时候，将矩阵层加到原参数上。大量实验表明，对于stable diffusion我们用Lora微调Unet网络注意力层可以取得良好的效果。\n",
+    "\n",
+    "## 动手实践\n",
+    "\n",
+    "首先我们需要下载代码和安装环境。"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "vscode": {
+     "languageId": "plaintext"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "git clone https://github.com/modelscope/modelscope.git\n",
+    "cd modelscope"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "然后我们执行脚本，开始dreambooth和lora的训练和推理。"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "vscode": {
+     "languageId": "plaintext"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "bash examples/pytorch/stable_diffusion/dreambooth/run_train_dreambooth.sh"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "vscode": {
+     "languageId": "plaintext"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "bash examples/pytorch/stable_diffusion/lora/run_train_lora.sh"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}