fix ms model

diffusers_video.py

@@ -1,13 +1,19 @@
 import torch
 from diffusers import DiffusionPipeline
 from diffusers.utils import export_to_video
+import time
 
 model_path = "/mnt/models/AI-ModelScope/text-to-video-ms-1.7b"
 pipe = DiffusionPipeline.from_pretrained(
-            model_path, torch_dtype=torch.float16, variant="fp16"
+            model_path, torch_dtype=torch.float32
             )
-pipe.enable_model_cpu_offload()       # 省显存
-pipe.enable_vae_slicing()
+# pipe.enable_model_cpu_offload()       # 省显存
+# pipe.enable_vae_slicing()
+pipe.to("cuda")
 
+start = time.time()
+print("Start generating")
 frames = pipe("Spiderman is surfing", num_frames=16).frames[0]
+end = time.time()
+print(f"elapsed: {end - start}")
 export_to_video(frames, "output.mp4")    # 默认约2秒/8fps

main.py

@@ -54,8 +54,9 @@ def build_pipeline(model_path: str, device: str = "cuda", dtype=torch.float16, m
         pipe = pipeline('text-to-video-synthesis', model_path, device=device)
     elif model_type == "text-to-video-ms":
         pipe = DiffusionPipeline.from_pretrained(model_path, torch_dtype=dtype)
-        pipe.enable_model_cpu_offload()       # 省显存
-        pipe.enable_vae_slicing()
+        pipe.to(device)
+        # pipe.enable_model_cpu_offload()       # 省显存
+        # pipe.enable_vae_slicing()
     else:
         raise ValueError(f"不支持的模型类型: {model_type}")
     return pipe

test_ms.sh

@@ -1 +1 @@
-python3 main.py --model "/mnt/models/AI-ModelScope/text-to-video-ms-1.7b" --json  "dataset.json" --results "results.json" --outdir "output" --device cuda --dtype fp16 --model_type "text-to-video-ms"
+python3 main.py --model "/mnt/models/AI-ModelScope/text-to-video-ms-1.7b" --json  "dataset.json" --results "results.json" --outdir "output" --device cuda --dtype fp32 --model_type "text-to-video-ms"