Switching model from huggingface to modelscope hub of efficient tuning (#479)

modelscope/models/multi_modal/efficient_diffusion_tuning/efficient_stable_diffusion.py

@@ -15,6 +15,7 @@ from diffusers.models import cross_attention
 from diffusers.utils import deprecation_utils
 from transformers import CLIPTextModel, CLIPTokenizer
 
+from modelscope import snapshot_download
 from modelscope.metainfo import Models
 from modelscope.models import TorchModel
 from modelscope.models.builder import MODELS
@@ -56,7 +57,10 @@ class EfficientStableDiffusion(TorchModel):
         super().__init__(model_dir, *args, **kwargs)
         tuner_name = kwargs.pop('tuner_name', 'lora')
         pretrained_model_name_or_path = kwargs.pop(
-            'pretrained_model_name_or_path', 'runwayml/stable-diffusion-v1-5')
+            'pretrained_model_name_or_path',
+            'AI-ModelScope/stable-diffusion-v1-5')
+        pretrained_model_name_or_path = snapshot_download(
+            pretrained_model_name_or_path)
         tuner_config = kwargs.pop('tuner_config', None)
         pretrained_tuner = kwargs.pop('pretrained_tuner', None)
         revision = kwargs.pop('revision', None)

tests/pipelines/test_efficient_diffusion_tuning.py

@@ -13,39 +13,45 @@ class EfficientDiffusionTuningTest(unittest.TestCase):
     def setUp(self) -> None:
         self.task = Tasks.efficient_diffusion_tuning
 
-    @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_efficient_diffusion_tuning_lora_run_pipeline(self):
         model_id = 'damo/multi-modal_efficient-diffusion-tuning-lora'
+        model_revision = 'v1.0.2'
         inputs = {'prompt': 'pale golden rod circle with old lace background'}
-        edt_pipeline = pipeline(self.task, model_id)
+        edt_pipeline = pipeline(
+            self.task, model_id, model_revision=model_revision)
         result = edt_pipeline(inputs)
         print(f'Efficient-diffusion-tuning-lora output: {result}.')
 
-    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_efficient_diffusion_tuning_lora_load_model_from_pretrained(self):
         model_id = 'damo/multi-modal_efficient-diffusion-tuning-lora'
-        model = Model.from_pretrained(model_id)
+        model_revision = 'v1.0.2'
+        model = Model.from_pretrained(model_id, model_revision=model_revision)
         self.assertTrue(model.__class__ == EfficientStableDiffusion)
 
-    @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_efficient_diffusion_tuning_control_lora_run_pipeline(self):
         # TODO: to be fixed in the future
         model_id = 'damo/multi-modal_efficient-diffusion-tuning-control-lora'
+        model_revision = 'v1.0.2'
         inputs = {
             'prompt':
             'pale golden rod circle with old lace background',
             'cond':
             'data/test/images/efficient_diffusion_tuning_sd_control_lora_source.png'
         }
-        edt_pipeline = pipeline(self.task, model_id)
+        edt_pipeline = pipeline(
+            self.task, model_id, model_revision=model_revision)
         result = edt_pipeline(inputs)
         print(f'Efficient-diffusion-tuning-control-lora output: {result}.')
 
-    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_efficient_diffusion_tuning_control_lora_load_model_from_pretrained(
             self):
         model_id = 'damo/multi-modal_efficient-diffusion-tuning-control-lora'
-        model = Model.from_pretrained(model_id)
+        model_revision = 'v1.0.2'
+        model = Model.from_pretrained(model_id, model_revision=model_revision)
         self.assertTrue(model.__class__ == EfficientStableDiffusion)
 
 

tests/pipelines/test_efficient_diffusion_tuning_swift.py

@@ -16,14 +16,16 @@ class EfficientDiffusionTuningTestSwift(unittest.TestCase):
     def setUp(self) -> None:
         self.task = Tasks.efficient_diffusion_tuning
 
-    @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_efficient_diffusion_tuning_swift_lora_run_pipeline(self):
         model_id = 'damo/multi-modal_efficient-diffusion-tuning-swift-lora'
+        model_revision = 'v1.0.2'
         inputs = {
             'prompt':
             'a street scene with a cafe and a restaurant sign in anime style'
         }
-        sd_tuner_pipeline = pipeline(self.task, model_id)
+        sd_tuner_pipeline = pipeline(
+            self.task, model_id, model_revision=model_revision)
         result = sd_tuner_pipeline(inputs, generator_seed=0)
         output_image_path = tempfile.NamedTemporaryFile(suffix='.png').name
         cv2.imwrite(output_image_path, result['output_imgs'][0])
@@ -31,21 +33,24 @@ class EfficientDiffusionTuningTestSwift(unittest.TestCase):
             f'Efficient-diffusion-tuning-swift-lora output: {output_image_path}'
         )
 
-    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_efficient_diffusion_tuning_swift_lora_load_model_from_pretrained(
             self):
         model_id = 'damo/multi-modal_efficient-diffusion-tuning-swift-lora'
-        model = Model.from_pretrained(model_id)
+        model_revision = 'v1.0.2'
+        model = Model.from_pretrained(model_id, model_revision=model_revision)
         self.assertTrue(model.__class__ == EfficientStableDiffusion)
 
-    @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_efficient_diffusion_tuning_swift_adapter_run_pipeline(self):
         model_id = 'damo/multi-modal_efficient-diffusion-tuning-swift-adapter'
+        model_revision = 'v1.0.2'
         inputs = {
             'prompt':
             'a street scene with a cafe and a restaurant sign in anime style'
         }
-        sd_tuner_pipeline = pipeline(self.task, model_id)
+        sd_tuner_pipeline = pipeline(
+            self.task, model_id, model_revision=model_revision)
         result = sd_tuner_pipeline(inputs, generator_seed=0)
         output_image_path = tempfile.NamedTemporaryFile(suffix='.png').name
         cv2.imwrite(output_image_path, result['output_imgs'][0])
@@ -53,21 +58,24 @@ class EfficientDiffusionTuningTestSwift(unittest.TestCase):
             f'Efficient-diffusion-tuning-swift-adapter output: {output_image_path}'
         )
 
-    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_efficient_diffusion_tuning_swift_adapter_load_model_from_pretrained(
             self):
         model_id = 'damo/multi-modal_efficient-diffusion-tuning-swift-adapter'
-        model = Model.from_pretrained(model_id)
+        model_revision = 'v1.0.2'
+        model = Model.from_pretrained(model_id, model_revision=model_revision)
         self.assertTrue(model.__class__ == EfficientStableDiffusion)
 
-    @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_efficient_diffusion_tuning_swift_prompt_run_pipeline(self):
         model_id = 'damo/multi-modal_efficient-diffusion-tuning-swift-prompt'
+        model_revision = 'v1.0.2'
         inputs = {
             'prompt':
             'a street scene with a cafe and a restaurant sign in anime style'
         }
-        sd_tuner_pipeline = pipeline(self.task, model_id)
+        sd_tuner_pipeline = pipeline(
+            self.task, model_id, model_revision=model_revision)
         result = sd_tuner_pipeline(inputs, generator_seed=0)
         output_image_path = tempfile.NamedTemporaryFile(suffix='.png').name
         cv2.imwrite(output_image_path, result['output_imgs'][0])
@@ -75,11 +83,12 @@ class EfficientDiffusionTuningTestSwift(unittest.TestCase):
             f'Efficient-diffusion-tuning-swift-prompt output: {output_image_path}'
         )
 
-    @unittest.skipUnless(test_level() >= 2, 'skip test in current test level')
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_efficient_diffusion_tuning_swift_prompt_load_model_from_pretrained(
             self):
         model_id = 'damo/multi-modal_efficient-diffusion-tuning-swift-prompt'
-        model = Model.from_pretrained(model_id)
+        model_revision = 'v1.0.2'
+        model = Model.from_pretrained(model_id, model_revision=model_revision)
         self.assertTrue(model.__class__ == EfficientStableDiffusion)
 
 

tests/trainers/test_efficient_diffusion_tuning_trainer.py

@@ -42,6 +42,7 @@ class TestEfficientDiffusionTuningTrainer(unittest.TestCase):
     @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
     def test_efficient_diffusion_tuning_lora_train(self):
         model_id = 'damo/multi-modal_efficient-diffusion-tuning-lora'
+        model_revision = 'v1.0.2'
 
         def cfg_modify_fn(cfg):
             cfg.train.max_epochs = self.max_epochs
@@ -51,6 +52,7 @@ class TestEfficientDiffusionTuningTrainer(unittest.TestCase):
 
         kwargs = dict(
             model=model_id,
+            model_revision=model_revision,
             work_dir=self.tmp_dir,
             train_dataset=self.train_dataset,
             eval_dataset=self.eval_dataset,
@@ -70,6 +72,7 @@ class TestEfficientDiffusionTuningTrainer(unittest.TestCase):
     @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
     def test_efficient_diffusion_tuning_lora_eval(self):
         model_id = 'damo/multi-modal_efficient-diffusion-tuning-lora'
+        model_revision = 'v1.0.2'
 
         def cfg_modify_fn(cfg):
             cfg.model.inference = False
@@ -77,6 +80,7 @@ class TestEfficientDiffusionTuningTrainer(unittest.TestCase):
 
         kwargs = dict(
             model=model_id,
+            model_revision=model_revision,
             work_dir=self.tmp_dir,
             train_dataset=None,
             eval_dataset=self.eval_dataset,
@@ -90,6 +94,7 @@ class TestEfficientDiffusionTuningTrainer(unittest.TestCase):
     @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
     def test_efficient_diffusion_tuning_control_lora_train(self):
         model_id = 'damo/multi-modal_efficient-diffusion-tuning-control-lora'
+        model_revision = 'v1.0.2'
 
         def cfg_modify_fn(cfg):
             cfg.train.max_epochs = self.max_epochs
@@ -99,6 +104,7 @@ class TestEfficientDiffusionTuningTrainer(unittest.TestCase):
 
         kwargs = dict(
             model=model_id,
+            model_revision=model_revision,
             work_dir=self.tmp_dir,
             train_dataset=self.train_dataset,
             eval_dataset=self.eval_dataset,
@@ -119,6 +125,7 @@ class TestEfficientDiffusionTuningTrainer(unittest.TestCase):
     @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
     def test_efficient_diffusion_tuning_control_lora_eval(self):
         model_id = 'damo/multi-modal_efficient-diffusion-tuning-control-lora'
+        model_revision = 'v1.0.2'
 
         def cfg_modify_fn(cfg):
             cfg.model.inference = False
@@ -126,6 +133,7 @@ class TestEfficientDiffusionTuningTrainer(unittest.TestCase):
 
         kwargs = dict(
             model=model_id,
+            model_revision=model_revision,
             work_dir=self.tmp_dir,
             train_dataset=None,
             eval_dataset=self.eval_dataset,

tests/trainers/test_efficient_diffusion_tuning_trainer_swift.py

@@ -33,9 +33,10 @@ class TestEfficientDiffusionTuningTrainerSwift(unittest.TestCase):
         shutil.rmtree(self.tmp_dir)
         super().tearDown()
 
-    @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_efficient_diffusion_tuning_swift_lora_train(self):
         model_id = 'damo/multi-modal_efficient-diffusion-tuning-swift-lora'
+        model_revision = 'v1.0.2'
 
         def cfg_modify_fn(cfg):
             cfg.train.max_epochs = self.max_epochs
@@ -47,6 +48,7 @@ class TestEfficientDiffusionTuningTrainerSwift(unittest.TestCase):
 
         kwargs = dict(
             model=model_id,
+            model_revision=model_revision,
             work_dir=self.tmp_dir,
             train_dataset=self.train_dataset,
             cfg_modify_fn=cfg_modify_fn)
@@ -60,9 +62,10 @@ class TestEfficientDiffusionTuningTrainerSwift(unittest.TestCase):
         self.assertIn(f'{trainer.timestamp}.log.json', results_files)
         self.assertIn(f'epoch_{self.max_epochs}.pth', results_files)
 
-    @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_efficient_diffusion_tuning_swift_adapter_train(self):
         model_id = 'damo/multi-modal_efficient-diffusion-tuning-swift-adapter'
+        model_revision = 'v1.0.2'
 
         def cfg_modify_fn(cfg):
             cfg.train.max_epochs = self.max_epochs
@@ -74,6 +77,7 @@ class TestEfficientDiffusionTuningTrainerSwift(unittest.TestCase):
 
         kwargs = dict(
             model=model_id,
+            model_revision=model_revision,
             work_dir=self.tmp_dir,
             train_dataset=self.train_dataset,
             cfg_modify_fn=cfg_modify_fn)
@@ -87,9 +91,10 @@ class TestEfficientDiffusionTuningTrainerSwift(unittest.TestCase):
         self.assertIn(f'{trainer.timestamp}.log.json', results_files)
         self.assertIn(f'epoch_{self.max_epochs}.pth', results_files)
 
-    @unittest.skipUnless(test_level() >= 1, 'skip test in current test level')
+    @unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
     def test_efficient_diffusion_tuning_swift_prompt_train(self):
         model_id = 'damo/multi-modal_efficient-diffusion-tuning-swift-prompt'
+        model_revision = 'v1.0.2'
 
         def cfg_modify_fn(cfg):
             cfg.train.max_epochs = self.max_epochs
@@ -101,6 +106,7 @@ class TestEfficientDiffusionTuningTrainerSwift(unittest.TestCase):
 
         kwargs = dict(
             model=model_id,
+            model_revision=model_revision,
             work_dir=self.tmp_dir,
             train_dataset=self.train_dataset,
             cfg_modify_fn=cfg_modify_fn)