From 6fb340e7f88634d7b731464e29c794ddd5d15c67 Mon Sep 17 00:00:00 2001 From: XDUWQ <1300964705@qq.com> Date: Mon, 24 Jul 2023 19:17:49 +0800 Subject: [PATCH 1/5] add lora_rank for lora stable diffusion --- .../lora/finetune_stable_diffusion_lora.py | 7 +++++++ .../pytorch/stable_diffusion/lora/run_train_lora.sh | 1 + .../lora_diffusion/lora_diffusion_trainer.py | 11 ++++++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/examples/pytorch/stable_diffusion/lora/finetune_stable_diffusion_lora.py b/examples/pytorch/stable_diffusion/lora/finetune_stable_diffusion_lora.py index 8ad3a28b..b878b5f8 100644 --- a/examples/pytorch/stable_diffusion/lora/finetune_stable_diffusion_lora.py +++ b/examples/pytorch/stable_diffusion/lora/finetune_stable_diffusion_lora.py @@ -18,6 +18,12 @@ class StableDiffusionLoraArguments(TrainingArgs): default='dog', metadata={ 'help': 'The pipeline prompt.', }) + + lora_rank: int = field( + default=4, metadata={ + 'help': 'The rank size of lora intermediate linear.', + } + ) training_args = StableDiffusionLoraArguments( @@ -59,6 +65,7 @@ kwargs = dict( work_dir=training_args.work_dir, train_dataset=train_dataset, eval_dataset=validation_dataset, + lora_rank=args.lora_rank, cfg_modify_fn=cfg_modify_fn) # build trainer and training diff --git a/examples/pytorch/stable_diffusion/lora/run_train_lora.sh b/examples/pytorch/stable_diffusion/lora/run_train_lora.sh index 876a2475..570c36af 100644 --- a/examples/pytorch/stable_diffusion/lora/run_train_lora.sh +++ b/examples/pytorch/stable_diffusion/lora/run_train_lora.sh @@ -5,6 +5,7 @@ PYTHONPATH=. torchrun examples/pytorch/stable_diffusion/lora/finetune_stable_dif --work_dir './tmp/lora_diffusion' \ --train_dataset_name 'buptwq/lora-stable-diffusion-finetune' \ --max_epochs 100 \ + --lora_rank 8 \ --save_ckpt_strategy 'by_epoch' \ --logging_interval 1 \ --train.dataloader.workers_per_gpu 0 \ diff --git a/modelscope/trainers/multi_modal/lora_diffusion/lora_diffusion_trainer.py b/modelscope/trainers/multi_modal/lora_diffusion/lora_diffusion_trainer.py index 99351fef..7c6644bd 100644 --- a/modelscope/trainers/multi_modal/lora_diffusion/lora_diffusion_trainer.py +++ b/modelscope/trainers/multi_modal/lora_diffusion/lora_diffusion_trainer.py @@ -34,6 +34,14 @@ class LoraDiffusionTrainer(EpochBasedTrainer): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + """Lora trainers for fine-tuning stable diffusion + + Args: + lora_rank: The rank size of lora intermediate linear. + + """ + lora_rank = kwargs.pop('lora_rank', 4) + # set lora save checkpoint processor ckpt_hook = list( filter(lambda hook: isinstance(hook, CheckpointHook), @@ -59,7 +67,8 @@ class LoraDiffusionTrainer(EpochBasedTrainer): lora_attn_procs[name] = LoRAAttnProcessor( hidden_size=hidden_size, - cross_attention_dim=cross_attention_dim) + cross_attention_dim=cross_attention_dim, + rank=lora_rank) self.model.unet.set_attn_processor(lora_attn_procs) From eb24e23d1993225a1f2781c3bdc9d7401c0edce5 Mon Sep 17 00:00:00 2001 From: XDUWQ <1300964705@qq.com> Date: Mon, 24 Jul 2023 19:24:52 +0800 Subject: [PATCH 2/5] add lora_rank for lora stable diffusion --- examples/pytorch/stable_diffusion/lora/run_train_lora.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pytorch/stable_diffusion/lora/run_train_lora.sh b/examples/pytorch/stable_diffusion/lora/run_train_lora.sh index 570c36af..bf62f833 100644 --- a/examples/pytorch/stable_diffusion/lora/run_train_lora.sh +++ b/examples/pytorch/stable_diffusion/lora/run_train_lora.sh @@ -5,7 +5,7 @@ PYTHONPATH=. torchrun examples/pytorch/stable_diffusion/lora/finetune_stable_dif --work_dir './tmp/lora_diffusion' \ --train_dataset_name 'buptwq/lora-stable-diffusion-finetune' \ --max_epochs 100 \ - --lora_rank 8 \ + --lora_rank 4 \ --save_ckpt_strategy 'by_epoch' \ --logging_interval 1 \ --train.dataloader.workers_per_gpu 0 \ From bc93e2dc96febadc6f285af1c5642c19871e1cea Mon Sep 17 00:00:00 2001 From: XDUWQ <1300964705@qq.com> Date: Mon, 24 Jul 2023 19:32:04 +0800 Subject: [PATCH 3/5] add lora_rank for lora stable diffusion --- .../stable_diffusion/lora/finetune_stable_diffusion_lora.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/pytorch/stable_diffusion/lora/finetune_stable_diffusion_lora.py b/examples/pytorch/stable_diffusion/lora/finetune_stable_diffusion_lora.py index b878b5f8..e15aa444 100644 --- a/examples/pytorch/stable_diffusion/lora/finetune_stable_diffusion_lora.py +++ b/examples/pytorch/stable_diffusion/lora/finetune_stable_diffusion_lora.py @@ -22,8 +22,7 @@ class StableDiffusionLoraArguments(TrainingArgs): lora_rank: int = field( default=4, metadata={ 'help': 'The rank size of lora intermediate linear.', - } - ) + }) training_args = StableDiffusionLoraArguments( From 426f55d57b8932aeb41aa1941f2172475542e8ca Mon Sep 17 00:00:00 2001 From: XDUWQ <1300964705@qq.com> Date: Mon, 24 Jul 2023 19:43:20 +0800 Subject: [PATCH 4/5] add lora_rank for lora stable diffusion --- .../stable_diffusion/lora/finetune_stable_diffusion_lora.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pytorch/stable_diffusion/lora/finetune_stable_diffusion_lora.py b/examples/pytorch/stable_diffusion/lora/finetune_stable_diffusion_lora.py index e15aa444..325cc7ae 100644 --- a/examples/pytorch/stable_diffusion/lora/finetune_stable_diffusion_lora.py +++ b/examples/pytorch/stable_diffusion/lora/finetune_stable_diffusion_lora.py @@ -18,7 +18,7 @@ class StableDiffusionLoraArguments(TrainingArgs): default='dog', metadata={ 'help': 'The pipeline prompt.', }) - + lora_rank: int = field( default=4, metadata={ 'help': 'The rank size of lora intermediate linear.', From 8e157cfa1500d34f386d29a63e8d330c5d229a64 Mon Sep 17 00:00:00 2001 From: XDUWQ <1300964705@qq.com> Date: Mon, 24 Jul 2023 22:10:52 +0800 Subject: [PATCH 5/5] precommit --- .../stable_diffusion/lora/finetune_stable_diffusion_lora.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/pytorch/stable_diffusion/lora/finetune_stable_diffusion_lora.py b/examples/pytorch/stable_diffusion/lora/finetune_stable_diffusion_lora.py index 325cc7ae..6001af48 100644 --- a/examples/pytorch/stable_diffusion/lora/finetune_stable_diffusion_lora.py +++ b/examples/pytorch/stable_diffusion/lora/finetune_stable_diffusion_lora.py @@ -20,7 +20,8 @@ class StableDiffusionLoraArguments(TrainingArgs): }) lora_rank: int = field( - default=4, metadata={ + default=4, + metadata={ 'help': 'The rank size of lora intermediate linear.', })