Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+llama2-extracted.pdf filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+# dotenv
+.env

README.md

@@ -1,12 +1,13 @@
 ---
-title: Pdfsearch
-emoji: 👀
-colorFrom: yellow
-colorTo: indigo
+title: pdfsearch
+app_file: web_demo.py
 sdk: gradio
-sdk_version: 4.19.2
-app_file: app.py
-pinned: false
+sdk_version: 4.15.0
 ---
+# chatpdf
+RAG homework
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+运行:
+```
+python web_demo.py
+```

import gradio as gr.py

@@ -0,0 +1,12 @@
+import gradio as gr
+
+def greet(lala, intensity):
+    return "Hello, " + lala + "!" * int(intensity)
+
+demo = gr.Interface(
+    fn=greet,
+    inputs=["text", "slider"],
+    outputs=["text"],
+)
+
+demo.launch()

llama2-extracted.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a965aaca87d5f56e7ab43abc562a7205d3d22f72dce49e8dac1bad3abd5f114
+size 1238532

openai_utils.py

@@ -0,0 +1,25 @@
+import openai
+import os
+# 加载环境变量
+from dotenv import load_dotenv, find_dotenv
+
+_ = load_dotenv(find_dotenv())  # 读取本地 .env 文件，里面定义了 OPENAI_API_KEY
+
+openai.api_key = os.getenv('OPENAI_API_KEY')
+
+
+def get_completion(prompt, context, model="gpt-3.5-turbo"):
+    """封装 openai 接口"""
+    messages = context + [{"role": "user", "content": prompt}]
+    response = openai.ChatCompletion.create(
+        model=model,
+        messages=messages,
+        temperature=0,  # 模型输出的随机性，0 表示随机性最小
+    )
+    return response.choices[0].message["content"]
+
+
+def get_embedding(text, model="text-embedding-3-small"):#text-embedding-ada-002
+    """封装 OpenAI 的 Embedding 模型接口"""
+    #return openai.Embedding.create(input=[text], model=model)['data'][0]['embedding']
+    return openai.Embedding.create(input=[text], model=model).data[0].embedding

pdf_utils.py

@@ -0,0 +1,27 @@
+from pdfminer.high_level import extract_pages
+from pdfminer.layout import LTTextContainer
+
+def extract_text_from_pdf(filename,page_numbers=None,min_line_length=10):
+    """从 PDF 文件中（按指定页码）提取文字"""
+    paragraphs = []
+    buffer = ''
+    full_text = ''
+    # 提取全部文本
+    for i, page_layout in enumerate(extract_pages(filename)):
+        # 如果指定了页码范围，跳过范围外的页
+        if page_numbers is not None and i not in page_numbers:
+            continue
+        for element in page_layout:
+            if isinstance(element, LTTextContainer):
+                full_text += element.get_text() + '\n'
+    # 按空行分隔，将文本重新组织成段落
+    lines = full_text.split('\n')
+    for text in lines:
+        if len(text) >= min_line_length:
+            buffer += (' '+text) if not text.endswith('-') else text.strip('-')
+        elif buffer:
+            paragraphs.append(buffer)
+            buffer = ''
+    if buffer:
+        paragraphs.append(buffer)
+    return paragraphs

prompt_utils.py

@@ -0,0 +1,28 @@
+prompt_template = """
+你是一个问答机器人。
+你的任务是根据下述给定的已知信息回答用户问题。
+确保你的回复完全依据下述已知信息。不要编造答案。
+如果下述已知信息不足以回答用户的问题，请直接回复"我无法回答您的问题"。
+
+已知信息:
+__INFO__
+
+用户问：
+__QUERY__
+
+请用中文回答用户问题。
+"""
+
+
+def build_prompt(template=prompt_template, **kwargs):
+    """将 Prompt 模板赋值"""
+    prompt = template
+    for k, v in kwargs.items():
+        if isinstance(v, str):
+            val = v
+        elif isinstance(v, list) and all(isinstance(elem, str) for elem in v):
+            val = '\n'.join(v)
+        else:
+            val = str(v)
+        prompt = prompt.replace(f"__{k.upper()}__", val)
+    return prompt

requirements.txt

@@ -0,0 +1,6 @@
+openai==0.28.1
+python-dotenv
+pdfminer.six
+nltk==3.8.1
+chromadb==0.4.15
+gradio

text_utils.py

@@ -0,0 +1,26 @@
+from nltk.tokenize import sent_tokenize
+import json
+
+def split_text(paragraphs,chunk_size=300,overlap_size=100):
+    """按指定 chunk_size 和 overlap_size 交叠割文本"""
+    sentences = [s.strip() for p in paragraphs for s in sent_tokenize(p)]
+    chunks = []
+    i= 0
+    while i < len(sentences):
+        chunk = sentences[i]
+        overlap = ''
+        prev_len = 0
+        prev = i - 1
+        # 向前计算重叠部分
+        while prev >= 0 and len(sentences[prev])+len(overlap) <= overlap_size:
+            overlap = sentences[prev] + ' ' + overlap
+            prev -= 1
+        chunk = overlap+chunk
+        next = i + 1
+        # 向后计算当前chunk
+        while next < len(sentences) and len(sentences[next])+len(chunk) <= chunk_size:
+            chunk = chunk + ' ' + sentences[next]
+            next += 1
+        chunks.append(chunk)
+        i = next
+    return chunks

vectordb_utils.py

@@ -0,0 +1,28 @@
+import chromadb
+from chromadb.config import Settings
+from openai_utils import get_embedding
+
+
+class InMemoryVecDB:
+
+    def __init__(self, name="demo"):
+        self.chroma_client = chromadb.Client(Settings(allow_reset=True))
+        self.chroma_client.reset()
+        self.name = name
+        self.collection = self.chroma_client.get_or_create_collection(name=name)
+
+    def add_documents(self, documents):
+        self.collection.add(
+            embeddings=[get_embedding(doc) for doc in documents],
+            documents=documents,
+            metadatas=[{"source": self.name} for _ in documents],
+            ids=[f"id_{i}" for i in range(len(documents))]
+        )
+
+    def search(self, query, top_n):
+        """检索向量数据库"""
+        results = self.collection.query(
+            query_embeddings=[get_embedding(query)],
+            n_results=top_n
+        )
+        return results['documents'][0]

web_demo.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+# coding=utf-8
+import gradio as gr
+from openai_utils import get_completion
+from prompt_utils import build_prompt
+from vectordb_utils import InMemoryVecDB
+from pdf_utils import extract_text_from_pdf
+from text_utils import split_text
+
+vec_db = InMemoryVecDB()  # 创建一个chromadb数据库对象
+
+# init_db函数用于初始化数据库。它首先从PDF文件中提取文本，然后将文本分割成一系列的段落，最后将这些段落添加到vec_db中。
+def init_db(file):
+    paragraphs = extract_text_from_pdf(file.name)
+    documents = split_text(paragraphs, 500, 100)
+    vec_db.add_documents(documents)
+
+# 调用chat函数时，先对用户输入的文本内容在vec_db中进行检索，然后构建一个提示Prompt，最后调用大模型，获得检索回复。
+def chat(user_input, chatbot, context, search_field):
+    search_results = vec_db.search(user_input, 3)   # 在向量数据库中搜索用户输入的文本
+    search_field = "\n\n".join(search_results)  # 将search_results列表中的所有元素连接成一个字符串，每个元素之间用两个换行符("\n\n")分隔.如果search_results是['result1', 'result2', 'result3']，那么"\n\n".join(search_results)的结果就是'result1\n\nresult2\n\nresult3'。这样，每个搜索结果都会在新的一行开始，使得输出更加清晰易读。
+    prompt = build_prompt(info = search_results, query = user_input) # 构建一个提示 Prompt
+    response = get_completion(prompt, context)  # 调用大模型，获得检索回复
+    chatbot.append((user_input, response))      # 将用户输入和检索回复添加到聊天机器人的对话中
+    context.append({'role': 'user', 'content': user_input})     # 将用户的问题添加到user角色下 
+    context.append({'role': 'assistant', 'content': response})  # 将检索回复添加到assistant角色下
+    return "", chatbot, context, search_field   # 返回空字符串，聊天机器人，context和检索结果
+
+# 重置聊天机器人的状态
+def reset_state():
+    return [], [], "", ""
+
+
+def main():
+    with gr.Blocks() as demo:
+        gr.HTML("""<h1 align="center">PDF内容检索器</h1>""")
+        gr.Markdown("本demo为您提供了一个简单的界面，用于从PDF文件中提取文本，并使用OpenAI的大模型来检索相关的信息。")
+
+        with gr.Row():
+            with gr.Column():
+                fileCtrl = gr.File(label="上传文件", file_types=[',pdf'])
+
+        with gr.Row():
+            with gr.Column(scale=2):
+                chatbot = gr.Chatbot()  # 创建一个Gradio聊天机器人对象
+            with gr.Column(scale=2):
+                # gr.HTML("""<h4>检索结果</h4>""")
+                search_field = gr.Textbox(show_label=True, label="检索结果", info="本检索采用的是单一RAG方式", placeholder="空...", lines=10)
+                user_input = gr.Textbox(show_label=True, label="用户输入", placeholder="请点击此处输入...", lines=3)
+                with gr.Row():
+                    submitBtn = gr.Button("提 交", variant="primary", size="lg")
+                    emptyBtn = gr.Button("清 空", size="sm", variant="secondary")
+
+        context = gr.State([])
+
+        # 当点击提交按钮时，调用chat函数
+        submitBtn.click(chat, [user_input, chatbot, context, search_field],
+                        [user_input, chatbot, context, search_field])
+        # 当点击清空按钮时，调用reset_state函数
+        emptyBtn.click(reset_state, outputs=[chatbot, context, user_input, search_field])
+
+        fileCtrl.upload(init_db, inputs = [fileCtrl])
+
+    demo.queue().launch(share=True, server_name='0.0.0.0', server_port=8080, inbrowser=True)
+
+
+if __name__ == "__main__":
+    main()