improve sd xl

load_ckpt.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+from diffusers import StableDiffusionPipeline, KDPM2DiscreteScheduler, StableDiffusionImg2ImgPipeline, HeunDiscreteScheduler, KDPM2AncestralDiscreteScheduler, DDIMScheduler
+import time
+import os
+from huggingface_hub import HfApi
+# from compel import Compel
+import torch
+import sys
+from pathlib import Path
+import requests
+from PIL import Image
+from io import BytesIO
+
+path = sys.argv[1]
+
+api = HfApi()
+start_time = time.time()
+pipe = StableDiffusionPipeline.from_ckpt(path, torch_dtype=torch.float16)
+import ipdb; ipdb.set_trace()
+
+pipe = pipe.to("cuda")
+
+prompt = "A lion in galaxies, spirals, nebulae, stars, smoke, iridescent, intricate detail, octane render, 8k"
+
+# rompts = ["a cat playing with a ball++ in the forest", "a cat playing with a ball++ in the forest", "a cat playing with a ball-- in the forest"]
+
+# prompt_embeds = torch.cat([compel.build_conditioning_tensor(prompt) for prompt in prompts])
+
+# generator = [torch.Generator(device="cuda").manual_seed(0) for _ in range(prompt_embeds.shape[0])]
+#
+# url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg"
+# 
+# response = requests.get(url)
+# image = Image.open(BytesIO(response.content)).convert("RGB")
+# image.thumbnail((768, 768))
+#
+
+for TIMESTEP_TYPE in ["trailing", "leading"]:
+    for RESCALE_BETAS_ZEROS_SNR in [True, False]:
+        for GUIDANCE_RESCALE in [0,0, 0.7]:
+
+            pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config, timestep_spacing=TIMESTEP_TYPE, rescale_betas_zero_snr=RESCALE_BETAS_ZEROS_SNR)
+            generator = torch.Generator(device="cpu").manual_seed(0)
+            images = pipe(prompt=prompt, generator=generator, num_images_per_prompt=4, num_inference_steps=40, guidance_rescale=GUIDANCE_RESCALE).images
+
+            for i, image in enumerate(images):
+                file_name = f"bb_{i}_{TIMESTEP_TYPE}_{str(int(RESCALE_BETAS_ZEROS_SNR))}_{GUIDANCE_RESCALE}"
+                path = os.path.join(Path.home(), "images", f"{file_name}.png")
+                image.save(path)
+
+                api.upload_file(
+                    path_or_fileobj=path,
+                    path_in_repo=path.split("/")[-1],
+                    repo_id="patrickvonplaten/images",
+                    repo_type="dataset",
+                )
+                print(f"https://huggingface.co/datasets/patrickvonplaten/images/blob/main/{file_name}.png")

run_local.py

@@ -11,16 +11,17 @@ import requests
 from PIL import Image
 from io import BytesIO
 
-# path = sys.argv[1]
-path = "runwayml/stable-diffusion-v1-5"
-path = "ptx0/pseudo-journey-v2"
+path = sys.argv[1]
+# path = "ptx0/pseudo-journey-v2"
 # path = "stabilityai/stable-diffusion-2-1"
 
 api = HfApi()
 start_time = time.time()
 pipe = StableDiffusionPipeline.from_pretrained(path, torch_dtype=torch.float16)
+
+pipe.unet = torch.compile(pipe.unet)
+
 # pipe = StableDiffusionImg2ImgPipeline.from_pretrained(path, torch_dtype=torch.float16, safety_checker=None)
-# pipe.scheduler = KDPM2AncestralDiscreteScheduler.from_config(pipe.scheduler.config)
 
 # compel = Compel(tokenizer=pipe.tokenizer, text_encoder=pipe.text_encoder)
 
@@ -36,29 +37,25 @@ prompt = "A lion in galaxies, spirals, nebulae, stars, smoke, iridescent, intric
 # generator = [torch.Generator(device="cuda").manual_seed(0) for _ in range(prompt_embeds.shape[0])]
 #
 # url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg"
-# 
+ 
 # response = requests.get(url)
 # image = Image.open(BytesIO(response.content)).convert("RGB")
 # image.thumbnail((768, 768))
-#
 
-for TIMESTEP_TYPE in ["trailing", "leading"]:
-    for RESCALE_BETAS_ZEROS_SNR in [True, False]:
-        for GUIDANCE_RESCALE in [0,0, 0.7]:
 
-            pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config, timestep_spacing=TIMESTEP_TYPE, rescale_betas_zero_snr=RESCALE_BETAS_ZEROS_SNR)
-            generator = torch.Generator(device="cpu").manual_seed(0)
-            images = pipe(prompt=prompt, generator=generator, num_images_per_prompt=4, num_inference_steps=40, guidance_rescale=GUIDANCE_RESCALE).images
+generator = torch.Generator(device="cpu").manual_seed(0)
+# images = pipe(prompt=prompt, image=image, generator=generator, num_images_per_prompt=4, num_inference_steps=25).images
+images = pipe(prompt=prompt, generator=generator, num_images_per_prompt=4, num_inference_steps=25).images
 
-            for i, image in enumerate(images):
-                file_name = f"bb_{i}_{TIMESTEP_TYPE}_{str(int(RESCALE_BETAS_ZEROS_SNR))}_{GUIDANCE_RESCALE}"
-                path = os.path.join(Path.home(), "images", f"{file_name}.png")
-                image.save(path)
+for i, image in enumerate(images):
+    file_name = f"bb_1_{i}"
+    path = os.path.join(Path.home(), "images", f"{file_name}.png")
+    image.save(path)
 
-                api.upload_file(
-                    path_or_fileobj=path,
-                    path_in_repo=path.split("/")[-1],
-                    repo_id="patrickvonplaten/images",
-                    repo_type="dataset",
-                )
-                print(f"https://huggingface.co/datasets/patrickvonplaten/images/blob/main/{file_name}.png")
+    api.upload_file(
+        path_or_fileobj=path,
+        path_in_repo=path.split("/")[-1],
+        repo_id="patrickvonplaten/images",
+        repo_type="dataset",
+    )
+    print(f"https://huggingface.co/datasets/patrickvonplaten/images/blob/main/{file_name}.png")

run_local_xl.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 from diffusers import DiffusionPipeline, EulerDiscreteScheduler, StableDiffusionPipeline, KDPM2DiscreteScheduler, StableDiffusionImg2ImgPipeline, HeunDiscreteScheduler, KDPM2AncestralDiscreteScheduler, DDIMScheduler
+from diffusers import StableDiffusionXLPipeline, StableDiffusionXLImg2ImgPipeline
 import time
 from pytorch_lightning import seed_everything
 import os
@@ -17,20 +18,22 @@ start_time = time.time()
 
 use_refiner = bool(int(sys.argv[1]))
 
-pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-0.9", torch_dtype=torch.float16, variant="fp16", use_safetensors=True)
+# pipe_1 = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-0.9", torch_dtype=torch.float16, variant="fp16", use_safetensors=True)
+pipe = StableDiffusionXLPipeline.from_single_file("https://huggingface.co/nichijoufan777/stable-diffusion-xl-base-0.9/blob/main/sd_xl_base_0.9.safetensors", torch_dtype=torch.float16, use_safetensors=True)
 pipe.to("cuda")
 # pipe.enable_model_cpu_offload()
 
 if use_refiner:
-    refiner = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-refiner-0.9", torch_dtype=torch.float16)
-    refiner.enable_model_cpu_offload()
+    refiner = StableDiffusionXLImg2ImgPipeline.from_single_file("https://huggingface.co/nichijoufan777/stable-diffusion-xl-refiner-0.9/blob/main/sd_xl_refiner_0.9.safetensors", torch_dtype=torch.float16)
+    refiner.to("cuda")
 
 prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k"
 seed_everything(0)
 image = pipe(prompt=prompt, output_type="latent" if use_refiner else "pil").images[0]
+# image = pipe(prompt=prompt, output_type="latent" if use_refiner else "pil").images[0]
 
 if use_refiner:
-    image = refiner(prompt=prompt, image=image[None, :]).imagas[0]
+    image = refiner(prompt=prompt, image=image[None, :]).images[0]
 
 # pipe.unet.to(memory_format=torch.channels_last)
 # pipe.unet = torch.compile(pipe.unet, mode="reduce-overhead", fullgraph=True)

run_safety.py

@@ -0,0 +1 @@
+#!/usr/bin/env python3

run_shape.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+import torch
+import numpy as np
+from huggingface_hub import HfApi
+
+from diffusers import ShapEPipeline
+from diffusers.utils import export_to_gif
+
+api = HfApi()
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+batch_size = 1
+guidance_scale = 15.0
+prompt = "a red table"
+prompt = "A chair that looks like an avocado"
+torch.manual_seed(0)
+
+repo = "openai/shap-e"
+pipe = ShapEPipeline.from_pretrained(repo)
+pipe = pipe.to(device)
+
+generator = torch.Generator(device="cuda").manual_seed(0)
+
+prompts = [
+    "A chair that looks like an avocado",
+    "An airplane that looks like a banana",
+    "A spaceship",
+    "A birthday cupcake",
+    "A chair that looks like a tree",
+    "A green boot",
+    "A penguin",
+    "Ube ice cream cone",
+    "A bowl of vegetables",
+]
+
+for prompt in prompts:
+    images = pipe(
+        prompt, 
+        num_images_per_prompt=batch_size, 
+        generator=generator, 
+        guidance_scale=guidance_scale,
+        num_inference_steps=64, 
+        frame_size=256, 
+        output_type='pil'
+    ).images
+
+    path = f"/home/patrick/images/{'_'.join(prompt.split())}.gif"
+    export_to_gif(images[0], path)
+
+
+    api.upload_file(
+        path_or_fileobj=path,
+        path_in_repo=path.split("/")[-1],
+        repo_id="patrickvonplaten/images",
+        repo_type="dataset",
+    )
+    print(f"https://huggingface.co/datasets/patrickvonplaten/images/blob/main/{path.split('/')[-1]}")

run_video.py

@@ -5,13 +5,16 @@ from diffusers.utils import export_to_video
 from PIL import Image
 
 # Make sure CUDA has < 13GB VRAM
-torch.cuda.set_per_process_memory_fraction(0.5)
+# torch.cuda.set_per_process_memory_fraction(0.5)
 
 pipe = DiffusionPipeline.from_pretrained("cerspense/zeroscope_v2_576w", torch_dtype=torch.float16)
 pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
-pipe.enable_model_cpu_offload()
+# pipe.enable_model_cpu_offload()
+pipe.to("cuda")
 pipe.enable_vae_slicing()
 
+pipe.unet = torch.compile(pipe.unet, mode="reduce-overhead", fullgraph=True)
+
 prompt = "Darth Vader is surfing on waves"
 video_frames = pipe(prompt, num_inference_steps=40, height=320, width=576, num_frames=36).frames
 video_path = export_to_video(video_frames, output_video_path="/home/patrick/videos/video_576_darth_vader_36.mp4")

speed_up_scheduler.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+import torch
+from diffusers import DiffusionPipeline
+from diffusers import EulerAncestralDiscreteScheduler
+
+import cProfile
+import pstats
+import io
+from pstats import SortKey
+
+path = 'stabilityai/stable-diffusion-2-1-base'
+prompt = "Women standing on a mountain top"
+
+torch.set_grad_enabled(False)
+torch.backends.cudnn.benchmark = True
+
+with torch.inference_mode():
+    pipe = DiffusionPipeline.from_pretrained(path, torch_dtype=torch.float16, safety_checker=None, requires_safety_checker=False)
+    pipe.to('cuda')
+    pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config)
+    pipe.unet.to(device='cuda', dtype=torch.float16, memory_format=torch.channels_last)
+
+    for bi in range(7):
+        if bi == 2:      # Start profiler on 3rd image
+            ob = cProfile.Profile()
+            ob.enable()
+        images = pipe(prompt=prompt, width=512, height=512, num_inference_steps=20, num_images_per_prompt=1).images
+    ob.disable()
+    sec = io.StringIO()
+    sortby = SortKey.TIME
+    ps = pstats.Stats(ob, stream=sec).sort_stats(sortby)
+    ps.print_stats()
+    print(sec.getvalue()[0:1000])