Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2026-07-13 22:08:46 +02:00
Code Issues Packages Projects Releases Wiki Activity

add parameters height and width for text-to-video

Browse Source 
Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/13171907
This commit is contained in:
baiguan.yt
2023-07-14 16:22:10 +08:00
committed by wenmeng.zwm
parent 41cbb8e393
commit ceac129c6b
3 changed files with 24 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
modelscope/models/multi_modal/video_synthesis/text_to_video_synthesis_model.py
View File

@@ -148,7 +148,8 @@ class TextToVideoSynthesis(Model):
with torch.no_grad():
num_sample = 1 # here let b = 1
max_frames = self.config.model.model_args.max_frames
latent_h, latent_w = 32, 32
latent_h, latent_w = input['out_height'] // 8, input[
'out_width'] // 8
with amp.autocast(enabled=True):
x0 = self.diffusion.ddim_sample_loop(
noise=torch.randn(num_sample, 4, max_frames, latent_h,

9
modelscope/pipelines/multi_modal/text_to_video_synthesis_pipeline.py
View File

@@ -52,7 +52,14 @@ class TextToVideoSynthesisPipeline(Pipeline):
text_emb_zero = self.model.clip_encoder('')
if self.model.config.model.model_args.tiny_gpu == 1:
self.model.clip_encoder.to('cpu')
return {'text_emb': text_emb, 'text_emb_zero': text_emb_zero}
out_height = input['height'] if 'height' in input else 256
out_width = input['width'] if 'height' in input else 256
return {
'text_emb': text_emb,
'text_emb_zero': text_emb_zero,
'out_height': out_height,
'out_width': out_width
}
def forward(self, input: Dict[str, Any],
**forward_params) -> Dict[str, Any]:

14
tests/pipelines/test_text_to_video_synthesis.py
View File

@@ -18,6 +18,12 @@ class TextToVideoSynthesisTest(unittest.TestCase):
'text': 'A panda eating bamboo on a rock.',
}
test_text_height_width = {
'text': 'A panda eating bamboo on a rock.',
'out_height': 256,
'out_width': 256,
}
@unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
def test_run_with_model_from_modelhub(self):
pipe_line_text_to_video_synthesis = pipeline(
@@ -26,6 +32,14 @@ class TextToVideoSynthesisTest(unittest.TestCase):
self.test_text)[OutputKeys.OUTPUT_VIDEO]
print(output_video_path)
@unittest.skipUnless(test_level() >= 0, 'skip test in current test level')
def test_run_modelhub_user_control(self):
pipe_line_text_to_video_synthesis = pipeline(
task=self.task, model=self.model_id)
output_video_path = pipe_line_text_to_video_synthesis(
self.test_text_height_width)[OutputKeys.OUTPUT_VIDEO]
print(output_video_path)
if __name__ == '__main__':
unittest.main()