introduce control net from diffusers

app.py

@@ -42,24 +42,7 @@ Since Visual ChatGPT is a text language model, Visual ChatGPT must use tools to
 The thoughts and observations are only visible for Visual ChatGPT, Visual ChatGPT should remember to repeat important information in the final response for Human. 
 Thought: Do I need to use a tool? {agent_scratchpad}"""
 
-import subprocess
-
-def execute_cmd(cmd):
-    output = subprocess.check_output(cmd, shell=True)
-    return output
-
-execute_cmd('ln -s ControlNet/ldm ./ldm')
-execute_cmd('ln -s ControlNet/cldm ./cldm')
-execute_cmd('ln -s ControlNet/annotator ./annotator')
-print(execute_cmd('nvidia-smi'))
-print(execute_cmd('nvcc -V'))
-
-from diffusers import StableDiffusionPipeline
-from diffusers import StableDiffusionInpaintPipeline
-from diffusers import StableDiffusionInstructPix2PixPipeline, EulerAncestralDiscreteScheduler
 from visual_foundation_models import *
-from omegaconf import OmegaConf
-from ldm.util import instantiate_from_config
 from langchain.agents.initialize import initialize_agent
 from langchain.agents.tools import Tool
 from langchain.chains.conversation.memory import ConversationBufferMemory
@@ -68,10 +51,6 @@ from langchain.vectorstores import Weaviate
 import re
 import gradio as gr
 
-try:
-    os.mkdir('./image')
-except OSError as error:
-    print(error)
 
 def cut_dialogue_history(history_memory, keep_last_n_words=500):
     tokens = history_memory.split()
@@ -87,29 +66,6 @@ def cut_dialogue_history(history_memory, keep_last_n_words=500):
             paragraphs = paragraphs[1:]
         return '\n' + '\n'.join(paragraphs)
 
-def get_new_image_name(org_img_name, func_name="update"):
-    head_tail = os.path.split(org_img_name)
-    head = head_tail[0]
-    tail = head_tail[1]
-    name_split = tail.split('.')[0].split('_')
-    this_new_uuid = str(uuid.uuid4())[0:4]
-    if len(name_split) == 1:
-        most_org_file_name = name_split[0]
-        recent_prev_file_name = name_split[0]
-        new_file_name = '{}_{}_{}_{}.png'.format(this_new_uuid, func_name, recent_prev_file_name, most_org_file_name)
-    else:
-        assert len(name_split) == 4
-        most_org_file_name = name_split[3]
-        recent_prev_file_name = name_split[0]
-        new_file_name = '{}_{}_{}_{}.png'.format(this_new_uuid, func_name, recent_prev_file_name, most_org_file_name)
-    return os.path.join(head, new_file_name)
-
-def create_model(config_path, device):
-    config = OmegaConf.load(config_path)
-    OmegaConf.update(config, "model.params.cond_stage_config.params.device", device)
-    model = instantiate_from_config(config.model).cpu()
-    print(f'Loaded model config from [{config_path}]')
-    return model
 
 class ConversationBot:
     def __init__(self):

visual_foundation_models.py

@@ -1,6 +1,3 @@
-import os
-
-import diffusers.utils
 from diffusers import StableDiffusionPipeline
 from diffusers import StableDiffusionInpaintPipeline
 from diffusers import StableDiffusionInstructPix2PixPipeline, EulerAncestralDiscreteScheduler
@@ -10,23 +7,15 @@ from controlnet_aux import OpenposeDetector, MLSDdetector, HEDdetector
 from transformers import AutoModelForCausalLM, AutoTokenizer, CLIPSegProcessor, CLIPSegForImageSegmentation
 from transformers import pipeline, BlipProcessor, BlipForConditionalGeneration, BlipForQuestionAnswering
 from transformers import AutoImageProcessor, UperNetForSemanticSegmentation
-# from ldm.util import instantiate_from_config
-# from ControlNet.cldm.model import create_model, load_state_dict
-# from ControlNet.cldm.ddim_hacked import DDIMSampler
-# from ControlNet.annotator.canny import CannyDetector
-# from ControlNet.annotator.mlsd import MLSDdetector
-# from ControlNet.annotator.hed import HEDdetector, nms
-# from ControlNet.annotator.openpose import OpenposeDetector
-# from ControlNet.annotator.uniformer import UniformerDetector
-# from ControlNet.annotator.midas import MidasDetector
+
 from PIL import Image
 import torch
 import numpy as np
 import uuid
-import einops
 from pytorch_lightning import seed_everything
 import cv2
 import random
+import os
 
 def ade_palette():
     return [[120, 120, 120], [180, 120, 120], [6, 230, 230], [80, 50, 50],