From ceac129c6be286aaf50c1730a235c19a0138773f Mon Sep 17 00:00:00 2001 From: "baiguan.yt" Date: Fri, 14 Jul 2023 16:22:10 +0800 Subject: [PATCH] add parameters height and width for text-to-video Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/13171907 --- .../text_to_video_synthesis_model.py | 3 ++- .../text_to_video_synthesis_pipeline.py | 9 ++++++++- tests/pipelines/test_text_to_video_synthesis.py | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/modelscope/models/multi_modal/video_synthesis/text_to_video_synthesis_model.py b/modelscope/models/multi_modal/video_synthesis/text_to_video_synthesis_model.py index 1bcd6eda..93a7e3ba 100644 --- a/modelscope/models/multi_modal/video_synthesis/text_to_video_synthesis_model.py +++ b/modelscope/models/multi_modal/video_synthesis/text_to_video_synthesis_model.py @@ -148,7 +148,8 @@ class TextToVideoSynthesis(Model): with torch.no_grad(): num_sample = 1 # here let b = 1 max_frames = self.config.model.model_args.max_frames - latent_h, latent_w = 32, 32 + latent_h, latent_w = input['out_height'] // 8, input[ + 'out_width'] // 8 with amp.autocast(enabled=True): x0 = self.diffusion.ddim_sample_loop( noise=torch.randn(num_sample, 4, max_frames, latent_h, diff --git a/modelscope/pipelines/multi_modal/text_to_video_synthesis_pipeline.py b/modelscope/pipelines/multi_modal/text_to_video_synthesis_pipeline.py index 50e2437b..f3ff7cce 100644 --- a/modelscope/pipelines/multi_modal/text_to_video_synthesis_pipeline.py +++ b/modelscope/pipelines/multi_modal/text_to_video_synthesis_pipeline.py @@ -52,7 +52,14 @@ class TextToVideoSynthesisPipeline(Pipeline): text_emb_zero = self.model.clip_encoder('') if self.model.config.model.model_args.tiny_gpu == 1: self.model.clip_encoder.to('cpu') - return {'text_emb': text_emb, 'text_emb_zero': text_emb_zero} + out_height = input['height'] if 'height' in input else 256 + out_width = input['width'] if 'height' in input else 256 + return { + 'text_emb': text_emb, + 'text_emb_zero': text_emb_zero, + 'out_height': out_height, + 'out_width': out_width + } def forward(self, input: Dict[str, Any], **forward_params) -> Dict[str, Any]: diff --git a/tests/pipelines/test_text_to_video_synthesis.py b/tests/pipelines/test_text_to_video_synthesis.py index 97ef6089..d2216949 100644 --- a/tests/pipelines/test_text_to_video_synthesis.py +++ b/tests/pipelines/test_text_to_video_synthesis.py @@ -18,6 +18,12 @@ class TextToVideoSynthesisTest(unittest.TestCase): 'text': 'A panda eating bamboo on a rock.', } + test_text_height_width = { + 'text': 'A panda eating bamboo on a rock.', + 'out_height': 256, + 'out_width': 256, + } + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_run_with_model_from_modelhub(self): pipe_line_text_to_video_synthesis = pipeline( @@ -26,6 +32,14 @@ class TextToVideoSynthesisTest(unittest.TestCase): self.test_text)[OutputKeys.OUTPUT_VIDEO] print(output_video_path) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_run_modelhub_user_control(self): + pipe_line_text_to_video_synthesis = pipeline( + task=self.task, model=self.model_id) + output_video_path = pipe_line_text_to_video_synthesis( + self.test_text_height_width)[OutputKeys.OUTPUT_VIDEO] + print(output_video_path) + if __name__ == '__main__': unittest.main()