import os
import tempfile
import time
import uuid
import cv2
import gradio as gr
import pymupdf
import spaces
import torch
from gradio_pdf import PDF
from loguru import logger
from PIL import Image
from transformers import AutoProcessor, VisionEncoderDecoderModel
from utils.utils import prepare_image, parse_layout_string, process_coordinates, ImageDimensions
# 读取外部CSS文件
def load_css():
css_path = os.path.join(os.path.dirname(__file__), "static", "styles.css")
if os.path.exists(css_path):
with open(css_path, "r", encoding="utf-8") as f:
return f.read()
return ""
# 全局变量存储模型
model = None
processor = None
tokenizer = None
# 自动初始化模型
@spaces.GPU
def initialize_model():
"""初始化 Hugging Face 模型"""
global model, processor, tokenizer
if model is None:
logger.info("Loading DOLPHIN model...")
model_id = "ByteDance/Dolphin"
# 加载处理器和模型
processor = AutoProcessor.from_pretrained(model_id)
model = VisionEncoderDecoderModel.from_pretrained(model_id)
model.eval()
# 设置设备和精度
device = "cuda" if torch.cuda.is_available() else "cpu"
model.to(device)
model = model.half() # 使用半精度
# 设置tokenizer
tokenizer = processor.tokenizer
logger.info(f"Model loaded successfully on {device}")
return "Model ready"
# 启动时自动初始化模型
logger.info("Initializing model at startup...")
try:
initialize_model()
logger.info("Model initialization completed")
except Exception as e:
logger.error(f"Model initialization failed: {e}")
# 模型将在首次使用时重新尝试初始化
# 模型推理函数
@spaces.GPU
def model_chat(prompt, image):
"""使用模型进行推理"""
global model, processor, tokenizer
# 确保模型已初始化
if model is None:
initialize_model()
# 检查是否为批处理
is_batch = isinstance(image, list)
if not is_batch:
images = [image]
prompts = [prompt]
else:
images = image
prompts = prompt if isinstance(prompt, list) else [prompt] * len(images)
# 准备图像
device = "cuda" if torch.cuda.is_available() else "cpu"
batch_inputs = processor(images, return_tensors="pt", padding=True)
batch_pixel_values = batch_inputs.pixel_values.half().to(device)
# 准备提示
prompts = [f"{p} " for p in prompts]
batch_prompt_inputs = tokenizer(
prompts,
add_special_tokens=False,
return_tensors="pt"
)
batch_prompt_ids = batch_prompt_inputs.input_ids.to(device)
batch_attention_mask = batch_prompt_inputs.attention_mask.to(device)
# 生成文本
outputs = model.generate(
pixel_values=batch_pixel_values,
decoder_input_ids=batch_prompt_ids,
decoder_attention_mask=batch_attention_mask,
min_length=1,
max_length=4096,
pad_token_id=tokenizer.pad_token_id,
eos_token_id=tokenizer.eos_token_id,
use_cache=True,
bad_words_ids=[[tokenizer.unk_token_id]],
return_dict_in_generate=True,
do_sample=False,
num_beams=1,
repetition_penalty=1.1
)
# 处理输出
sequences = tokenizer.batch_decode(outputs.sequences, skip_special_tokens=False)
# 清理提示文本
results = []
for i, sequence in enumerate(sequences):
cleaned = sequence.replace(prompts[i], "").replace("", "").replace("", "").strip()
results.append(cleaned)
# 返回单个结果或批处理结果
if not is_batch:
return results[0]
return results
# 处理元素批次
@spaces.GPU
def process_element_batch(elements, prompt, max_batch_size=16):
"""处理同类型元素的批次"""
results = []
# 确定批次大小
batch_size = min(len(elements), max_batch_size)
# 分批处理
for i in range(0, len(elements), batch_size):
batch_elements = elements[i:i+batch_size]
crops_list = [elem["crop"] for elem in batch_elements]
# 使用相同的提示
prompts_list = [prompt] * len(crops_list)
# 批量推理
batch_results = model_chat(prompts_list, crops_list)
# 添加结果
for j, result in enumerate(batch_results):
elem = batch_elements[j]
results.append({
"label": elem["label"],
"bbox": elem["bbox"],
"text": result.strip(),
"reading_order": elem["reading_order"],
})
return results
# 清理临时文件
def cleanup_temp_file(file_path):
"""安全地删除临时文件"""
try:
if file_path and os.path.exists(file_path):
os.unlink(file_path)
except Exception as e:
logger.warning(f"Failed to cleanup temp file {file_path}: {e}")
def to_pdf(file_path):
"""将输入文件转换为PDF格式"""
if file_path is None:
return None
with pymupdf.open(file_path) as f:
if f.is_pdf:
return file_path
else:
pdf_bytes = f.convert_to_pdf()
# 使用临时文件而不是保存到磁盘
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp_file:
tmp_file.write(pdf_bytes)
return tmp_file.name
@spaces.GPU(duration=120)
def process_document(file_path):
"""处理文档的主要函数 - 集成完整的推理逻辑"""
if file_path is None:
return "", "", {}, {}
start_time = time.time()
original_file_path = file_path
# 确保模型已初始化
if model is None:
initialize_model()
# 转换为PDF(如果需要)
converted_file_path = to_pdf(file_path)
temp_file_created = converted_file_path != original_file_path
try:
logger.info(f"Processing document: {file_path}")
# 处理页面
recognition_results = process_page(converted_file_path)
# 生成Markdown内容
md_content = generate_markdown(recognition_results)
# 计算处理时间
processing_time = time.time() - start_time
debug_info = {
"original_file": original_file_path,
"converted_file": converted_file_path,
"temp_file_created": temp_file_created,
"status": "success",
"processing_time": f"{processing_time:.2f}s",
"total_elements": len(recognition_results)
}
processing_data = {
"pages": [{"elements": recognition_results}],
"total_elements": len(recognition_results),
"processing_time": f"{processing_time:.2f}s"
}
logger.info(f"Document processed successfully in {processing_time:.2f}s")
return md_content, md_content, processing_data, debug_info
except Exception as e:
logger.error(f"Error processing document: {str(e)}")
error_info = {
"original_file": original_file_path,
"converted_file": converted_file_path,
"temp_file_created": temp_file_created,
"status": "error",
"error": str(e)
}
return f"# 处理错误\n\n处理文档时发生错误: {str(e)}", "", {}, error_info
finally:
# 清理临时文件
if temp_file_created:
cleanup_temp_file(converted_file_path)
def process_page(image_path):
"""处理单页文档"""
# 阶段1: 页面级布局解析
pil_image = Image.open(image_path).convert("RGB")
layout_output = model_chat("Parse the reading order of this document.", pil_image)
# 阶段2: 元素级内容解析
padded_image, dims = prepare_image(pil_image)
recognition_results = process_elements(layout_output, padded_image, dims)
return recognition_results
def process_elements(layout_results, padded_image, dims, max_batch_size=16):
"""解析所有文档元素"""
layout_results = parse_layout_string(layout_results)
# 分别存储不同类型的元素
text_elements = [] # 文本元素
table_elements = [] # 表格元素
figure_results = [] # 图像元素(无需处理)
previous_box = None
reading_order = 0
# 收集要处理的元素并按类型分组
for bbox, label in layout_results:
try:
# 调整坐标
x1, y1, x2, y2, orig_x1, orig_y1, orig_x2, orig_y2, previous_box = process_coordinates(
bbox, padded_image, dims, previous_box
)
# 裁剪并解析元素
cropped = padded_image[y1:y2, x1:x2]
if cropped.size > 0:
if label == "fig":
# 对于图像区域,直接添加空文本结果
figure_results.append(
{
"label": label,
"bbox": [orig_x1, orig_y1, orig_x2, orig_y2],
"text": "",
"reading_order": reading_order,
}
)
else:
# 准备元素进行解析
pil_crop = Image.fromarray(cv2.cvtColor(cropped, cv2.COLOR_BGR2RGB))
element_info = {
"crop": pil_crop,
"label": label,
"bbox": [orig_x1, orig_y1, orig_x2, orig_y2],
"reading_order": reading_order,
}
# 按类型分组
if label == "tab":
table_elements.append(element_info)
else: # 文本元素
text_elements.append(element_info)
reading_order += 1
except Exception as e:
logger.error(f"Error processing bbox with label {label}: {str(e)}")
continue
# 初始化结果列表
recognition_results = figure_results.copy()
# 处理文本元素(批量)
if text_elements:
text_results = process_element_batch(text_elements, "Read text in the image.", max_batch_size)
recognition_results.extend(text_results)
# 处理表格元素(批量)
if table_elements:
table_results = process_element_batch(table_elements, "Parse the table in the image.", max_batch_size)
recognition_results.extend(table_results)
# 按阅读顺序排序
recognition_results.sort(key=lambda x: x.get("reading_order", 0))
return recognition_results
def generate_markdown(recognition_results):
"""从识别结果生成Markdown内容"""
markdown_parts = []
for result in recognition_results:
text = result.get("text", "").strip()
label = result.get("label", "")
if text:
if label == "tab":
# 表格内容
markdown_parts.append(f"\n{text}\n")
else:
# 普通文本内容
markdown_parts.append(text)
return "\n\n".join(markdown_parts)
# LaTeX 渲染配置
latex_delimiters = [
{"left": "$$", "right": "$$", "display": True},
{"left": "$", "right": "$", "display": False},
{"left": "\\[", "right": "\\]", "display": True},
{"left": "\\(", "right": "\\)", "display": False},
]
# 加载自定义CSS
custom_css = load_css()
# 读取页面头部
with open("header.html", "r", encoding="utf-8") as file:
header = file.read()
# 创建 Gradio 界面
with gr.Blocks(css=custom_css, title="Dolphin Document Parser") as demo:
gr.HTML(header)
with gr.Row():
# 侧边栏 - 文件上传和控制
with gr.Column(scale=1, elem_classes="sidebar"):
# 文件上传组件
file = gr.File(
label="Choose PDF or image file",
file_types=[".pdf", ".png", ".jpeg", ".jpg"],
elem_id="file-upload"
)
gr.HTML("选择文件后,点击处理按钮开始解析
After selecting the file, click the Process button to start parsing")
with gr.Row(elem_classes="action-buttons"):
submit_btn = gr.Button("处理文档/Process Document", variant="primary")
clear_btn = gr.ClearButton(value="清空/Clear")
# 处理状态显示
status_display = gr.Textbox(
label="Processing Status",
value="Ready to process documents",
interactive=False,
max_lines=2
)
# 示例文件
example_root = os.path.join(os.path.dirname(__file__), "examples")
if os.path.exists(example_root):
gr.HTML("示例文件/Example Files")
example_files = [
os.path.join(example_root, f)
for f in os.listdir(example_root)
if not f.endswith(".py")
]
examples = gr.Examples(
examples=example_files,
inputs=file,
examples_per_page=10,
elem_id="example-files"
)
# 主体内容区域
with gr.Column(scale=7):
with gr.Row(elem_classes="main-content"):
# 预览面板
with gr.Column(scale=1, elem_classes="preview-panel"):
gr.HTML("文件预览/Preview")
pdf_show = PDF(label="", interactive=False, visible=True, height=600)
debug_output = gr.JSON(label="Debug Info", height=100)
# 输出面板
with gr.Column(scale=1, elem_classes="output-panel"):
with gr.Tabs():
with gr.Tab("Markdown [Render]"):
md_render = gr.Markdown(
label="",
height=700,
show_copy_button=True,
latex_delimiters=latex_delimiters,
line_breaks=True,
)
with gr.Tab("Markdown [Content]"):
md_content = gr.TextArea(lines=30, show_copy_button=True)
with gr.Tab("Processing Data"):
json_output = gr.JSON(label="", height=700)
# 事件处理
file.change(fn=to_pdf, inputs=file, outputs=pdf_show)
# 文档处理
def process_with_status(file_path):
"""处理文档并更新状态"""
if file_path is None:
return "", "", {}, {}, "Please select a file first"
# 更新状态为处理中
status = "Processing document..."
# 执行文档处理
md_render_result, md_content_result, json_result, debug_result = process_document(file_path)
# 更新完成状态
if "错误" in md_render_result:
status = "Processing failed - see debug info"
else:
status = "Processing completed successfully"
return md_render_result, md_content_result, json_result, debug_result, status
submit_btn.click(
fn=process_with_status,
inputs=[file],
outputs=[md_render, md_content, json_output, debug_output, status_display],
)
# 清空所有内容
def reset_all():
return None, None, "", "", {}, {}, "Ready to process documents"
clear_btn.click(
fn=reset_all,
inputs=[],
outputs=[file, pdf_show, md_render, md_content, json_output, debug_output, status_display]
)
# 启动应用
if __name__ == "__main__":
demo.launch()