Update qa_vector_store.py

qa_vector_store.py

@@ -15,10 +15,6 @@ def build_qa_vector_store(model_name, collection_name):
     df.columns = ['Question', 'Answer']
     original_len = len(df)
 
-    # 去除重複 QA 組合
-    df = df.drop_duplicates(subset=["Question", "Answer"]).reset_index(drop=True)
-    print(f"📊 原始資料筆數：{original_len}，去除重複後筆數：{len(df)}")
-
     questions = df['Question'].tolist()
     answers = df['Answer'].tolist()
     # 初始化模型
@@ -48,33 +44,10 @@ def build_qa_vector_store(model_name, collection_name):
     print(f"✅ 向量資料庫建立完成，共嵌入 {len(points)} 筆 QA。")
     client.scroll(collection_name=collection_name, limit=100)
 
-
-# build_qa_vector_store(model_name, collection_name)
-
-# %%
-# model_name = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
-# collection_name = model_name.split("/")[-1]  
-# client = QdrantClient(path="./qadrant_data")
-# count = client.count(collection_name=collection_name, exact=True).count
-# print(f"📦 Collection {collection_name} 中有 {count} 筆資料")
-# # %%
-# from collections import Counter
-
-# records = client.scroll(collection_name=collection_name, limit=1000)[0]
-# answers = [rec.payload["answer"] for rec in records]
-# duplicates = [item for item, count in Counter(answers).items() if count > 1]
-# print("重複答案數量：", len(duplicates))
-# print("部分重複答案：", duplicates[:5])
-
 def retrieve_and_rerank(query, model_name, collection_name, cross_encoder_model, score_threshold=0.6, search_top_k=25, rerank_top_k=5):
     from semantic_reranker import rerank_results
     model = SentenceModel(model_name)
     client = QdrantClient(path="./qadrant_data")
-
-    # 確認 collection 是否存在
-    if collection_name not in [c.name for c in client.get_collections().collections]:
-        print(f"⚠️ Collection {collection_name} 不存在，請先建立向量資料庫。")
-        return
         
     query_vector = model.encode(query, normalize_embeddings=True)
     results = client.search(
@@ -93,15 +66,6 @@ def retrieve_and_rerank(query, model_name, collection_name, cross_encoder_model,
         candidate_passages=retrieved_answers,
         top_k=rerank_top_k
     )   
-
-    if not results:
-        print("❌ 找不到相關答案。")
-    else:
-        for i, hit in enumerate(results):
-            print(f"✅ Top {i+1}：{hit.payload['answer']} (score={hit.score:.3f})")
-
-    for i, (answer, score) in enumerate(reranked):
-        print(f"🔥 Rerank Top {i+1}：{answer} (score={score:.3f})")
     return reranked
 
 
@@ -123,7 +87,6 @@ def generate_response_from_local_llm(query, passages, tokenizer, model, max_new_
 - 回答時平易近人，像和朋友交談一樣。
 - 精簡回答，避免冗長的解釋。
 回答："""
-    print(prompt)
     inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
     outputs = model.generate(
         **inputs,
@@ -135,27 +98,6 @@ def generate_response_from_local_llm(query, passages, tokenizer, model, max_new_
     decoded_output = tokenizer.decode(outputs[0], skip_special_tokens=True)    
 
     # 提取回答部分    
-    answer = decoded_output.split("回答：", 1)[-1].strip() if "回答：" in decoded_output else decoded_output
-    return answer
-#%%
-# from sentence_transformers import CrossEncoder
-# from transformers import AutoTokenizer, AutoModelForCausalLM
-# model_name = "sentence-transformers/paraphrase-multilingual-mpnet-base-v2"
-# collection_name = model_name.split("/")[-1]  
-# cross_encoder_model = CrossEncoder("cross-encoder/mmarco-mMiniLMv2-L12-H384-v1")
-# tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B", trust_remote_code=True)
-# model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2-0.5B", trust_remote_code=True)
-# # tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen1.5-7B-Chat", trust_remote_code=True)
-# # model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen1.5-7B-Chat", trust_remote_code=True)
-
-# #%%
-# user_query = "許智傑做過什麼壞事"
-# reranked = retrieve_and_rerank(user_query, model_name, collection_name, cross_encoder_model, score_threshold=0.6, search_top_k=20, rerank_top_k=5)
-# #%%
-# passages = [answer for answer, score in reranked]
-# answer = generate_response_from_local_llm(user_query, passages, tokenizer, model, max_new_tokens=256)
-# print("回答：", answer)
-
-
-
-# %%
+    answer = decoded_output.split("回答：", 1)[-1].strip() 
+    answer = answer + "大罷免!大成功!"
+    return answer