controlnet.py

import torch
from diffusers import (
    StableDiffusionControlNetImg2ImgPipeline,
    ControlNetModel,
    DDIMScheduler,
)

from PIL import Image


class QRControlNet:

    def __init__(self, device: str = "cuda"):

        torch_dtype = torch.float16 if device == "cuda" else torch.float32
        controlnet = ControlNetModel.from_pretrained(
            "DionTimmer/controlnet_qrcode-control_v1p_sd15", torch_dtype=torch_dtype
        )

        pipe = StableDiffusionControlNetImg2ImgPipeline.from_pretrained(
            "runwayml/stable-diffusion-v1-5",
            controlnet=controlnet,
            # safety_checker=None,
            torch_dtype=torch_dtype,
        ).to(device)

        if device == "cuda":
            pipe.enable_xformers_memory_efficient_attention()
            pipe.enable_model_cpu_offload()

        pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config)
        self.pipe = pipe

    def generate_image(
        self,
        source_image: Image,
        control_image: Image,
        prompt: str,
        negative_prompt: str,
        img_size=512,
        num_inference_steps: int = 50,
        guidance_scale: int = 20,
        controlnet_conditioning_scale: float = 3.0,
        strength=0.9,
        seed=42,
        **kwargs
    ):

        width = height = img_size
        generator = torch.manual_seed(seed)

        image = self.pipe(
            prompt=prompt,
            negative_prompt=negative_prompt,
            image=source_image,
            control_image=control_image,
            width=width,
            height=height,
            guidance_scale=guidance_scale,
            controlnet_conditioning_scale=controlnet_conditioning_scale,  # 3.0,
            generator=generator,
            strength=strength,
            num_inference_steps=num_inference_steps,
            **kwargs
        )

        return image.images[0]