lib/methods/sr.py

import os
from functools import partial
from glob import glob
from pathlib import Path as PythonPath

import cv2
import torchvision.transforms.functional as TvF
import torch
import torch.nn as nn
import numpy as np
from inspect import isfunction
from PIL import Image

from lib import smplfusion
from lib.smplfusion import share, router, attentionpatch, transformerpatch
from lib.utils.iimage import IImage
from lib.utils import poisson_blend
from lib.models.sd2_sr import predict_eps_from_z_and_v, predict_start_from_z_and_v


def refine_mask(hr_image, hr_mask, lr_image, sam_predictor):
    lr_mask = hr_mask.resize(512)

    x_min, y_min, rect_w, rect_h = cv2.boundingRect(lr_mask.data[0][:, :, 0])
    x_min = max(x_min - 1, 0)
    y_min = max(y_min - 1, 0)
    x_max = x_min + rect_w + 1
    y_max = y_min + rect_h + 1

    input_box = np.array([x_min, y_min, x_max, y_max])

    sam_predictor.set_image(hr_image.resize(512).data[0])
    masks, _, _ = sam_predictor.predict(
        point_coords=None,
        point_labels=None,
        box=input_box[None, :],
        multimask_output=True,
    )
    dilation_kernel = np.ones((13, 13))
    original_object_mask = (np.sum(masks, axis=0) > 0).astype(np.uint8)
    original_object_mask = cv2.dilate(original_object_mask, dilation_kernel)

    sam_predictor.set_image(lr_image.resize(512).data[0])
    masks, _, _ = sam_predictor.predict(
        point_coords=None,
        point_labels=None,
        box=input_box[None, :],
        multimask_output=True,
    )
    dilation_kernel = np.ones((3, 3))
    inpainted_object_mask = (np.sum(masks, axis=0) > 0).astype(np.uint8)
    inpainted_object_mask = cv2.dilate(inpainted_object_mask, dilation_kernel)

    lr_mask_masking = ((original_object_mask + inpainted_object_mask ) > 0).astype(np.uint8)
    new_mask = lr_mask.data[0] * lr_mask_masking[:, :, np.newaxis]
    new_mask = IImage(new_mask).resize(2048, resample = Image.BICUBIC)
    return new_mask


def run(
    ddim,
    sam_predictor,
    lr_image,
    hr_image,
    hr_mask,
    prompt = 'high resolution professional photo',
    noise_level=20,
    blend_output = True,
    blend_trick = True,
    dt = 50,
    seed = 1,
    guidance_scale = 7.5,
    negative_prompt = '',
    use_sam_mask = False
):
    hr_image_info = hr_image.info
    lr_image = IImage(lr_image)
    hr_image = IImage(hr_image).resize(2048)
    hr_mask = IImage(hr_mask).resize(2048)

    torch.manual_seed(seed)
    dtype = ddim.vae.encoder.conv_in.weight.dtype
    device = ddim.vae.encoder.conv_in.weight.device

    router.attention_forward = attentionpatch.default.forward_xformers
    router.basic_transformer_forward = transformerpatch.default.forward

    hr_image_orig = hr_image
    hr_mask_orig = hr_mask

    if use_sam_mask:
        with torch.no_grad():
            hr_mask = refine_mask(hr_image, hr_mask, lr_image, sam_predictor)

    orig_h, orig_w = hr_image.torch().shape[2], hr_image.torch().shape[3]
    hr_image = hr_image.padx(256, padding_mode='reflect')
    hr_mask = hr_mask.padx(256, padding_mode='reflect').dilate(19)
   
    lr_image = lr_image.padx(64, padding_mode='reflect').torch()
    lr_mask = hr_mask.resize((lr_image.shape[2:]), resample = Image.BICUBIC)
    lr_mask = lr_mask.alpha().torch(vmin=0).to(device)
    lr_mask = TvF.gaussian_blur(lr_mask, kernel_size=19)
                                     
    # encode hr image
    with torch.no_grad():
        hr_image = hr_image.torch().to(dtype=dtype, device=device)
        hr_z0 = ddim.vae.encode(hr_image).mean * ddim.config.scale_factor

    assert hr_z0.shape[2] == lr_image.shape[2]
    assert hr_z0.shape[3] == lr_image.shape[3]                                  
                                     
    with torch.no_grad():
        context = ddim.encoder.encode([negative_prompt, prompt])
    
    noise_level = torch.Tensor(1 * [noise_level]).to(device=device).long()
    unet_condition = lr_image.to(dtype=dtype, device=device, memory_format=torch.contiguous_format)
    unet_condition, noise_level = ddim.low_scale_model(unet_condition, noise_level=noise_level)

    with torch.autocast('cuda'), torch.no_grad():
        zt = torch.randn((1,4,unet_condition.shape[2], unet_condition.shape[3]))
        zt = zt.cuda().to(dtype=dtype, device=device)
        for index,t in enumerate(range(999, 0, -dt)):
            _zt = zt if unet_condition is None else torch.cat([zt, unet_condition], 1)
            eps_uncond, eps = ddim.unet(
                torch.cat([_zt, _zt]).to(dtype=dtype, device=device), 
                timesteps = torch.tensor([t, t]).to(device=device), 
                context = context,
                y=torch.cat([noise_level]*2)
            ).chunk(2)
            ts = torch.full((zt.shape[0],), t, device=device, dtype=torch.long)
            model_output = (eps_uncond + guidance_scale * (eps - eps_uncond))
            eps = predict_eps_from_z_and_v(ddim.schedule, zt, ts, model_output).to(dtype)
            z0 = predict_start_from_z_and_v(ddim.schedule, zt, ts, model_output).to(dtype)
            if blend_trick:
                z0 = z0 * lr_mask + hr_z0 * (1-lr_mask)
            zt = ddim.schedule.sqrt_alphas[t - dt] * z0 + ddim.schedule.sqrt_one_minus_alphas[t - dt] * eps

    with torch.no_grad():
        hr_result = ddim.vae.decode(z0.to(dtype) / ddim.config.scale_factor)
        # postprocess
        hr_result = (255 * ((hr_result + 1) / 2).clip(0, 1)).to(torch.uint8)
        hr_result = hr_result.cpu().permute(0, 2, 3, 1)[0].numpy()
        hr_result = hr_result[:orig_h, :orig_w, :]

    if blend_output:
        hr_result = poisson_blend(
            orig_img=hr_image_orig.data[0],
            fake_img=hr_result,
            mask=hr_mask_orig.alpha().data[0]
        )
    hr_result = Image.fromarray(hr_result)
    hr_result.info = hr_image_info  # save metadata
    return hr_result