demo.py

import argparse
import os
import torch
import torch.backends.cudnn as cudnn
from PIL import Image
from src.utils.vis import prob_to_mask
from huggingface_hub import hf_hub_download
from tools import load_model, process_image, post_process_output, get_masked_depth, get_point_cloud, removebg_crop

parser = argparse.ArgumentParser("Arguments for deploying a LaRI Demo")
parser.add_argument(
    "--image_path",
    type=str,
    default="assets/cole_hardware.png",
    help="input image name",
)

parser.add_argument(
    "--output_path",
    type=str,
    default="./results",
    help="path to save the image",
)

parser.add_argument(
    "--model_info_pm",
    type=str,
    default="LaRIModel(use_pretrained = 'moge_full', num_output_layer = 5, head_type = 'point')",
    help="Network parameters to load the model",
)

parser.add_argument(
    "--model_info_mask",
    type=str,
    default="DinoSegModel(use_pretrained = 'dinov2', dim_proj = 256, pretrained_path = '', num_output_layer = 4, output_type = 'ray_stop')",
    help="Network parameters to load the model",
)

parser.add_argument(
    "--ckpt_path_pm",
    type=str,
    default="lari_obj_16k_pointmap.pth",
    help="Path to pre-trained weights",
)

parser.add_argument(
    "--ckpt_path_mask",
    type=str,
    default="lari_obj_16k_seg.pth",
    help="Path to pre-trained weights",
)

parser.add_argument(
    "--resolution", type=int, default=512, help="Default model resolution"
)

parser.add_argument(
    "--is_remove_background", action="store_true", help="Automatically remove the background."
)

args = parser.parse_args()






device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
cudnn.benchmark = True

# === Load the model

model_path_pm = hf_hub_download(repo_id="ruili3/LaRI", filename=args.ckpt_path_pm, repo_type="model")
model_path_mask = hf_hub_download(repo_id="ruili3/LaRI", filename=args.ckpt_path_mask, repo_type="model")
# Load the model with pretrained weights.
model_pm = load_model(args.model_info_pm, model_path_pm, device)
model_mask = (
    load_model(args.model_info_mask, model_path_mask, device)
    if args.model_info_mask is not None
    else None
)

# === Image pre-processing
pil_input = Image.open(args.image_path)
if args.is_remove_background:
    pil_input = removebg_crop(pil_input) # remove background
input_tensor, ori_img_tensor, crop_coords, original_size = process_image(
    pil_input, resolution=512) # crop & resize to fit the model input size
input_tensor = input_tensor.to(device)


# === Run inference
with torch.no_grad():
    # lari map
    pred_dict = model_pm(input_tensor)
    lari_map = -pred_dict["pts3d"].squeeze(
        0
    )
    # mask
    if model_mask:
        pred_dict = model_mask(input_tensor)
        assert "seg_prob" in pred_dict
        valid_mask = prob_to_mask(pred_dict["seg_prob"].squeeze(0))  # H W L 1
    else:
        h, w, l, _ = lari_map.shape
        valid_mask = torch.new_ones((h, w, l, 1), device=lari_map.device)

# === crop & resize back to the original resolution
if original_size[0] != args.resolution or original_size[1] != args.resolution:
    lari_map = post_process_output(lari_map, crop_coords, original_size)  # H W L 3
    valid_mask = post_process_output(
        valid_mask.float(), crop_coords, original_size
    ).bool()  # H W L 1

max_n_layer = min(valid_mask.shape[-2], lari_map.shape[-2])
valid_mask = valid_mask[:, :, :max_n_layer, :]
lari_map = lari_map[:, :, :max_n_layer, :]


# === save output
os.makedirs(args.output_path, exist_ok=True)

for layer_id in range(max_n_layer):
    depth_pil = get_masked_depth(
        lari_map=lari_map, valid_mask=valid_mask, layer_id=layer_id
    )
    depth_pil.save(os.path.join(args.output_path, f"layered_depth_{layer_id}.jpg"))


# point cloud
glb_path, ply_path = get_point_cloud(
    lari_map, ori_img_tensor, valid_mask, first_layer_color="pseudo",
    target_folder=args.output_path
)

print("All results saved to `{}`.".format(args.output_path))