From d1478db172cbb1ffdccdbda73837b0fcf350238b Mon Sep 17 00:00:00 2001 From: XDUWQ <1300964705@qq.com> Date: Thu, 31 Aug 2023 10:11:48 +0800 Subject: [PATCH 01/11] fix crossattention of diffusers --- .../efficient_diffusion_tuning/efficient_stable_diffusion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py b/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py index 901c44d9..e9f136ce 100644 --- a/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py +++ b/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py @@ -11,7 +11,7 @@ import torch.nn.functional as F from diffusers import (AutoencoderKL, DDPMScheduler, DiffusionPipeline, DPMSolverMultistepScheduler, UNet2DConditionModel, utils) -from diffusers.models import cross_attention +from diffusers.models import attention from diffusers.utils import deprecation_utils from swift import AdapterConfig, LoRAConfig, PromptConfig, Swift from transformers import CLIPTextModel, CLIPTokenizer @@ -30,7 +30,7 @@ from .control_sd_lora import ControlLoRATuner utils.deprecate = lambda *arg, **kwargs: None deprecation_utils.deprecate = lambda *arg, **kwargs: None -cross_attention.deprecate = lambda *arg, **kwargs: None +attention.deprecate = lambda *arg, **kwargs: None __tuner_MAP__ = {'lora': LoRATuner, 'control_lora': ControlLoRATuner} From c217d293096bfa52822055bd160efd0388d0fe48 Mon Sep 17 00:00:00 2001 From: XDUWQ <1300964705@qq.com> Date: Thu, 31 Aug 2023 10:40:59 +0800 Subject: [PATCH 02/11] fix bugs --- .../efficient_diffusion_tuning/control_sd_lora.py | 7 ++++--- .../multi_modal/efficient_diffusion_tuning/sd_lora.py | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/modelscope/models/multi_modal/efficient_diffusion_tuning/control_sd_lora.py b/modelscope/models/multi_modal/efficient_diffusion_tuning/control_sd_lora.py index a2c53e24..aaa588d3 100644 --- a/modelscope/models/multi_modal/efficient_diffusion_tuning/control_sd_lora.py +++ b/modelscope/models/multi_modal/efficient_diffusion_tuning/control_sd_lora.py @@ -10,7 +10,8 @@ import torch import torch.nn as nn import torch.nn.functional as F from diffusers.configuration_utils import ConfigMixin, register_to_config -from diffusers.models.cross_attention import CrossAttention, LoRALinearLayer +from diffusers.models.attention_processor import Attention +from diffusers.models.lora import LoRALinearLayer from diffusers.models.modeling_utils import ModelMixin from diffusers.models.resnet import (Downsample2D, Upsample2D, downsample_2d, partial, upsample_2d) @@ -467,7 +468,7 @@ class ControlLoRACrossAttnProcessor(LoRACrossAttnProcessor): return control_states def __call__(self, - attn: CrossAttention, + attn: Attention, hidden_states, encoder_hidden_states=None, attention_mask=None, @@ -619,7 +620,7 @@ class ControlLoRACrossAttnProcessorV2(LoRACrossAttnProcessor): return control_states def __call__(self, - attn: CrossAttention, + attn: Attention, hidden_states, encoder_hidden_states=None, attention_mask=None, diff --git a/modelscope/models/multi_modal/efficient_diffusion_tuning/sd_lora.py b/modelscope/models/multi_modal/efficient_diffusion_tuning/sd_lora.py index feff05f4..306ca2b0 100644 --- a/modelscope/models/multi_modal/efficient_diffusion_tuning/sd_lora.py +++ b/modelscope/models/multi_modal/efficient_diffusion_tuning/sd_lora.py @@ -8,7 +8,8 @@ from typing import List, Tuple, Union import torch import torch.nn as nn from diffusers.configuration_utils import ConfigMixin, register_to_config -from diffusers.models.cross_attention import CrossAttention, LoRALinearLayer +from diffusers.models.attention_processor import Attention +from diffusers.models.lora import LoRALinearLayer from diffusers.models.modeling_utils import ModelMixin from diffusers.utils.outputs import BaseOutput @@ -84,7 +85,7 @@ class LoRACrossAttnProcessor(nn.Module): self.output_states_skipped = is_skipped def __call__(self, - attn: CrossAttention, + attn: Attention, hidden_states, encoder_hidden_states=None, attention_mask=None, From 6d13b58eae7e4447f5b3077d7ec2172bfb7e733d Mon Sep 17 00:00:00 2001 From: XDUWQ <1300964705@qq.com> Date: Thu, 31 Aug 2023 12:05:04 +0800 Subject: [PATCH 03/11] fix bugx bugs --- .../efficient_diffusion_tuning/efficient_stable_diffusion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py b/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py index e9f136ce..583d5ac0 100644 --- a/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py +++ b/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py @@ -139,7 +139,7 @@ class EfficientStableDiffusion(TorchModel): 320, 320, 640, 640, 1280, 1280, 1280, 1280, 1280, 640, 640, 640, 320, 320, 320 ], - module_layer_name= + target_modules= r'.*[down_blocks|up_blocks|mid_block]\.\d+\.attentions\.\d+\.transformer_blocks\.\d+$', embedding_pos=0, prompt_length=prompt_length, From a9de26f683f58b4e18f9b74e9500e5ff665a6559 Mon Sep 17 00:00:00 2001 From: XDUWQ <1300964705@qq.com> Date: Thu, 31 Aug 2023 12:56:30 +0800 Subject: [PATCH 04/11] fix bugs --- .../efficient_diffusion_tuning/efficient_stable_diffusion.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py b/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py index 583d5ac0..7a989070 100644 --- a/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py +++ b/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py @@ -113,7 +113,7 @@ class EfficientStableDiffusion(TorchModel): rank = tuner_config[ 'rank'] if tuner_config and 'rank' in tuner_config else 4 lora_config = LoRAConfig( - rank=rank, + r=rank, replace_modules=['to_q', 'to_k', 'to_v', 'to_out.0'], merge_weights=False, only_lora_trainable=False, @@ -126,7 +126,7 @@ class EfficientStableDiffusion(TorchModel): adapter_config = AdapterConfig( dim=-1, hidden_pos=0, - module_name=r'.*ff\.net\.2$', + target_modules=r'.*ff\.net\.2$', adapter_length=adapter_length, only_adapter_trainable=False, pretrained_weights=pretrained_tuner) @@ -143,7 +143,6 @@ class EfficientStableDiffusion(TorchModel): r'.*[down_blocks|up_blocks|mid_block]\.\d+\.attentions\.\d+\.transformer_blocks\.\d+$', embedding_pos=0, prompt_length=prompt_length, - only_prompt_trainable=False, attach_front=False, pretrained_weights=pretrained_tuner, extract_embedding=True) From 7715050047840d46814e333840aa788f974bffe8 Mon Sep 17 00:00:00 2001 From: XDUWQ <1300964705@qq.com> Date: Thu, 31 Aug 2023 16:59:18 +0800 Subject: [PATCH 05/11] fix bugs --- .../efficient_stable_diffusion.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py b/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py index 7a989070..d3919166 100644 --- a/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py +++ b/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py @@ -114,11 +114,10 @@ class EfficientStableDiffusion(TorchModel): 'rank'] if tuner_config and 'rank' in tuner_config else 4 lora_config = LoRAConfig( r=rank, - replace_modules=['to_q', 'to_k', 'to_v', 'to_out.0'], + target_modules=['to_q', 'to_k', 'to_v', 'to_out.0'], merge_weights=False, only_lora_trainable=False, - use_merged_linear=False, - pretrained_weights=pretrained_tuner) + use_merged_linear=False) self.unet = Swift.prepare_model(self.unet, lora_config) elif tuner_name == 'swift-adapter': adapter_length = tuner_config[ @@ -127,9 +126,7 @@ class EfficientStableDiffusion(TorchModel): dim=-1, hidden_pos=0, target_modules=r'.*ff\.net\.2$', - adapter_length=adapter_length, - only_adapter_trainable=False, - pretrained_weights=pretrained_tuner) + adapter_length=adapter_length) self.unet = Swift.prepare_model(self.unet, adapter_config) elif tuner_name == 'swift-prompt': prompt_length = tuner_config[ From 83ee5b9d111b1b516c44ff99394147fa728c8c61 Mon Sep 17 00:00:00 2001 From: XDUWQ <1300964705@qq.com> Date: Thu, 31 Aug 2023 19:12:49 +0800 Subject: [PATCH 06/11] fix bugs --- .../efficient_diffusion_tuning/efficient_stable_diffusion.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py b/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py index d3919166..c4f46cbf 100644 --- a/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py +++ b/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py @@ -141,7 +141,6 @@ class EfficientStableDiffusion(TorchModel): embedding_pos=0, prompt_length=prompt_length, attach_front=False, - pretrained_weights=pretrained_tuner, extract_embedding=True) self.unet = Swift.prepare_model(self.unet, prompt_config) elif tuner_name in ('lora', 'control_lora'): From da58f397e1acc115d56bd739c6daa3e4b4e3b099 Mon Sep 17 00:00:00 2001 From: XDUWQ <1300964705@qq.com> Date: Thu, 31 Aug 2023 19:24:17 +0800 Subject: [PATCH 07/11] fix bugs --- .../efficient_diffusion_tuning/efficient_stable_diffusion.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py b/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py index c4f46cbf..18f655c2 100644 --- a/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py +++ b/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py @@ -140,8 +140,7 @@ class EfficientStableDiffusion(TorchModel): r'.*[down_blocks|up_blocks|mid_block]\.\d+\.attentions\.\d+\.transformer_blocks\.\d+$', embedding_pos=0, prompt_length=prompt_length, - attach_front=False, - extract_embedding=True) + attach_front=False) self.unet = Swift.prepare_model(self.unet, prompt_config) elif tuner_name in ('lora', 'control_lora'): # if not set the config of control-tuner, we add the lora tuner directly to the original framework, From bb7a1c497e655be8b41794b2ee155afc1209bf90 Mon Sep 17 00:00:00 2001 From: XDUWQ <1300964705@qq.com> Date: Thu, 31 Aug 2023 20:17:29 +0800 Subject: [PATCH 08/11] fix bugs --- .../efficient_diffusion_tuning/efficient_stable_diffusion.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py b/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py index 18f655c2..2fcd1df8 100644 --- a/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py +++ b/modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py @@ -116,7 +116,6 @@ class EfficientStableDiffusion(TorchModel): r=rank, target_modules=['to_q', 'to_k', 'to_v', 'to_out.0'], merge_weights=False, - only_lora_trainable=False, use_merged_linear=False) self.unet = Swift.prepare_model(self.unet, lora_config) elif tuner_name == 'swift-adapter': From 3b0006f52c3cd4e049c9da01a185d58e07df354a Mon Sep 17 00:00:00 2001 From: XDUWQ <1300964705@qq.com> Date: Thu, 31 Aug 2023 20:46:41 +0800 Subject: [PATCH 09/11] skip tests --- .../test_efficient_diffusion_tuning_trainer_swift.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/trainers/test_efficient_diffusion_tuning_trainer_swift.py b/tests/trainers/test_efficient_diffusion_tuning_trainer_swift.py index 9e12335e..c661b8ee 100644 --- a/tests/trainers/test_efficient_diffusion_tuning_trainer_swift.py +++ b/tests/trainers/test_efficient_diffusion_tuning_trainer_swift.py @@ -33,7 +33,7 @@ class TestEfficientDiffusionTuningTrainerSwift(unittest.TestCase): shutil.rmtree(self.tmp_dir) super().tearDown() - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_efficient_diffusion_tuning_swift_lora_train(self): model_id = 'damo/multi-modal_efficient-diffusion-tuning-swift-lora' model_revision = 'v1.0.2' @@ -62,7 +62,7 @@ class TestEfficientDiffusionTuningTrainerSwift(unittest.TestCase): self.assertIn(f'{trainer.timestamp}.log.json', results_files) self.assertIn(f'epoch_{self.max_epochs}.pth', results_files) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_efficient_diffusion_tuning_swift_adapter_train(self): model_id = 'damo/multi-modal_efficient-diffusion-tuning-swift-adapter' model_revision = 'v1.0.2' @@ -91,7 +91,7 @@ class TestEfficientDiffusionTuningTrainerSwift(unittest.TestCase): self.assertIn(f'{trainer.timestamp}.log.json', results_files) self.assertIn(f'epoch_{self.max_epochs}.pth', results_files) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_efficient_diffusion_tuning_swift_prompt_train(self): model_id = 'damo/multi-modal_efficient-diffusion-tuning-swift-prompt' model_revision = 'v1.0.2' From c89de1a825be958029f08878a3162ef20c6cda9d Mon Sep 17 00:00:00 2001 From: XDUWQ <1300964705@qq.com> Date: Thu, 31 Aug 2023 21:46:12 +0800 Subject: [PATCH 10/11] skip tests --- .../test_efficient_diffusion_tuning_swift.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/pipelines/test_efficient_diffusion_tuning_swift.py b/tests/pipelines/test_efficient_diffusion_tuning_swift.py index 09b739a0..a2af7dec 100644 --- a/tests/pipelines/test_efficient_diffusion_tuning_swift.py +++ b/tests/pipelines/test_efficient_diffusion_tuning_swift.py @@ -16,7 +16,7 @@ class EfficientDiffusionTuningTestSwift(unittest.TestCase): def setUp(self) -> None: self.task = Tasks.efficient_diffusion_tuning - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_efficient_diffusion_tuning_swift_lora_run_pipeline(self): model_id = 'damo/multi-modal_efficient-diffusion-tuning-swift-lora' model_revision = 'v1.0.2' @@ -33,7 +33,7 @@ class EfficientDiffusionTuningTestSwift(unittest.TestCase): f'Efficient-diffusion-tuning-swift-lora output: {output_image_path}' ) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_efficient_diffusion_tuning_swift_lora_load_model_from_pretrained( self): model_id = 'damo/multi-modal_efficient-diffusion-tuning-swift-lora' @@ -41,7 +41,7 @@ class EfficientDiffusionTuningTestSwift(unittest.TestCase): model = Model.from_pretrained(model_id, model_revision=model_revision) self.assertTrue(model.__class__ == EfficientStableDiffusion) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_efficient_diffusion_tuning_swift_adapter_run_pipeline(self): model_id = 'damo/multi-modal_efficient-diffusion-tuning-swift-adapter' model_revision = 'v1.0.2' @@ -58,7 +58,7 @@ class EfficientDiffusionTuningTestSwift(unittest.TestCase): f'Efficient-diffusion-tuning-swift-adapter output: {output_image_path}' ) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_efficient_diffusion_tuning_swift_adapter_load_model_from_pretrained( self): model_id = 'damo/multi-modal_efficient-diffusion-tuning-swift-adapter' @@ -66,7 +66,7 @@ class EfficientDiffusionTuningTestSwift(unittest.TestCase): model = Model.from_pretrained(model_id, model_revision=model_revision) self.assertTrue(model.__class__ == EfficientStableDiffusion) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_efficient_diffusion_tuning_swift_prompt_run_pipeline(self): model_id = 'damo/multi-modal_efficient-diffusion-tuning-swift-prompt' model_revision = 'v1.0.2' @@ -83,7 +83,7 @@ class EfficientDiffusionTuningTestSwift(unittest.TestCase): f'Efficient-diffusion-tuning-swift-prompt output: {output_image_path}' ) - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_efficient_diffusion_tuning_swift_prompt_load_model_from_pretrained( self): model_id = 'damo/multi-modal_efficient-diffusion-tuning-swift-prompt' From cf43991981efbb45c5a95f8a90459ed66eec4f22 Mon Sep 17 00:00:00 2001 From: XDUWQ <1300964705@qq.com> Date: Thu, 31 Aug 2023 23:42:01 +0800 Subject: [PATCH 11/11] skip tests --- tests/export/test_export_stable_diffusion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/export/test_export_stable_diffusion.py b/tests/export/test_export_stable_diffusion.py index a2e20198..91a877da 100644 --- a/tests/export/test_export_stable_diffusion.py +++ b/tests/export/test_export_stable_diffusion.py @@ -20,7 +20,7 @@ class TestExportStableDiffusion(unittest.TestCase): os.makedirs(self.tmp_dir) self.model_id = 'AI-ModelScope/stable-diffusion-v1-5' - @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_export_stable_diffusion(self): model = Model.from_pretrained(self.model_id) Exporter.from_model(model).export_onnx(