Upload 4 files

app.py

@@ -2,7 +2,7 @@ import streamlit as st
 from generator import generate_response_from_document
 from retrieval import retrieve_documents_hybrid,find_query_dataset
 from evaluation import calculate_metrics
-from data_processing import load_recent_questions, save_recent_question
+from data_processing import load_recent_questions, save_recent_questions
 import time
 import matplotlib.pyplot as plt
 
@@ -22,26 +22,6 @@ st.markdown(
     unsafe_allow_html=True
 )
 
-# global retrieved_documents
-# retrieved_documents = []
-
-# global response 
-# response = ""
-
-# global time_taken_for_response
-# time_taken_for_response = 'N/A'
-
-# @st.cache_data
-# def load_data():
-#     load_data_from_faiss()
-
-# data_status = load_data()
-
-# Question Section
-st.subheader("Hi, What do you want to know today?")
-question = st.text_area("Enter your question:", placeholder="Type your question here...", height=100)
-question = question.strip()
-
 # # Submit Button
 # if st.button("Submit"):
 #     start_time = time.time()
@@ -70,48 +50,49 @@ question = question.strip()
 # with col2:
 #     st.text_area("Metrics:", value=metrics, height=100, disabled=True)
 
-if "retrieved_documents" not in st.session_state:
-    st.session_state.retrieved_documents = []
-if "response" not in st.session_state:
-    st.session_state.response = ""
-if "time_taken_for_response" not in st.session_state:
-    st.session_state.time_taken_for_response = "N/A"
-if "metrics" not in st.session_state:
-    st.session_state.metrics = {}
-if "query_dataset" not in st.session_state:
-    st.session_state.query_dataset = ''
+# Initialize session state
 if "recent_questions" not in st.session_state:
-    st.session_state.recent_questions = {}
+    st.session_state.recent_questions = load_recent_questions()
+
+if "last_question" not in st.session_state:
+    st.session_state.last_question = None
+
+if "response_time" not in st.session_state:
+    st.session_state.response_time = None
+
+if "retrieved_documents" not in st.session_state:
+    st.session_state.retrieved_documents = None
 
-st.session_state.recent_questions  = load_recent_questions()
-print(st.session_state.recent_questions )
+if "response" not in st.session_state:
+    st.session_state.response = None
 
-if st.session_state.recent_questions  and "questions" in st.session_state.recent_questions  and st.session_state.recent_questions ["questions"]:
-    recent_qns = list(reversed(st.session_state.recent_questions ["questions"]))
+if st.session_state.recent_questions:
+    recent_qns = list(reversed(st.session_state.recent_questions))
 
     print(recent_qns)
 
     # Display Recent Questions
-    st.sidebar.title("Recent Questions")
-    for q in recent_qns:  # Show latest first
-        st.sidebar.write(f"🔹 {q['question']}")
+    st.sidebar.title("Overall RMSE")
+    rmse_values = [q["metrics"]["rmse"] for q in recent_qns if "metrics" in q and "rmse" in q["metrics"]]
+    average_rmse = sum(rmse_values) / len(rmse_values) if rmse_values else 0
+    st.sidebar.write(f"📊 **Average RMSE:** {average_rmse:.4f}")
 
     st.sidebar.markdown("---")
+    
     st.sidebar.title("Analytics")
 
     # Extract response times and labels
-    response_time = [q['metrics']["response_time"] for q in recent_qns]
+    response_time = [q.get('metrics').get('response_time') for q in recent_qns]
     labels = [f"Q{i+1}" for i in range(len(response_time))]
 
     # Plot graph
-    fig, ax = plt.subplots()
-    ax.plot(labels, response_time, marker="o", linestyle="-", color="skyblue")
-    ax.set_xlabel("Recent Questions")
-    ax.set_ylabel("Time Taken for Response (seconds)")
-    ax.set_title("Response Time Analysis")
-
-    # Display the plot in the sidebar
-    st.sidebar.pyplot(fig)
+    if any(response_time):
+        fig, ax = plt.subplots()
+        ax.plot(labels, response_time, marker="o", linestyle="-", color="skyblue")
+        ax.set_xlabel("Recent Questions")
+        ax.set_ylabel("Time Taken for Response (seconds)")
+        ax.set_title("Response Time Analysis")
+        st.sidebar.pyplot(fig)
 
     st.sidebar.markdown("---")
 
@@ -122,52 +103,55 @@ if st.session_state.recent_questions  and "questions" in st.session_state.recent
 else:
     st.sidebar.title("No recent questions")
 
-if st.button("Submit"):
-    start_time = time.time()
-    st.session_state.metrics = {}
-    st.session_state.query_dataset =  find_query_dataset(question)
-    st.session_state.retrieved_documents = retrieve_documents_hybrid(question, st.session_state.query_dataset, 10)  
-    st.session_state.response = generate_response_from_document(question, st.session_state.retrieved_documents)
-    end_time = time.time()
-    st.session_state.time_taken_for_response = end_time - start_time
-
-    # Store in session state
-    # st.session_state.recent_questions.append({
-    #     "question": question,
-    #     "response_time": st.session_state.time_taken_for_response
-    # })
+# Question Section
+st.subheader("Hi, What do you want to know today?")
+question = st.text_area("Enter your question:", placeholder="Type your question here...", height=100)
+question = question.strip()
 
+if st.button("Submit"):
+    if question:
+        st.session_state.last_question = question
+
+        start_time = time.time()
+        st.session_state.metrics = {}
+        st.session_state.response = ""
+        st.session_state.query_dataset =  find_query_dataset(question)
+        st.session_state.retrieved_documents = retrieve_documents_hybrid(question, st.session_state.query_dataset, 10)  
+        st.session_state.response = generate_response_from_document(question, st.session_state.retrieved_documents)
+        end_time = time.time()
+        st.session_state.time_taken_for_response = end_time - start_time
+
+        # Check if question already exists
+        existing_questions = [q["question"] for q in st.session_state.recent_questions]
+        
+        if question not in existing_questions:
+            new_entry = {
+                    "question": question,
+                    "metrics": st.session_state.metrics
+                }
+            st.session_state.recent_questions.append(new_entry)
+            save_recent_questions(st.session_state.recent_questions)
+    else:
+        st.error("Please enter a question before submitting.")
+    
 # Display stored response
 st.subheader("Response")
 st.text_area("Generated Response:", value=st.session_state.response, height=150, disabled=True)
 
 col1, col2 = st.columns([1, 3])  # Creating two columns for button and metrics display
 
-# # Calculate Metrics Button
-# with col1:
-#     if st.button("Calculate Metrics"):
-#         metrics = calculate_metrics(question, st.session_state.response, st.session_state.retrieved_documents, st.session_state.time_taken_for_response)
-#     else:
-#         metrics = {}
-
-# with col2:
-#     #st.text_area("Metrics:", value=metrics, height=100, disabled=True)
-#     st.json(metrics)
-
-
 # Calculate Metrics Button
 with col1:
     if st.button("Show Metrics"):     
         st.session_state.metrics = calculate_metrics(question, st.session_state.query_dataset, st.session_state.response, st.session_state.retrieved_documents, st.session_state.time_taken_for_response)
-        metrics_ = st.session_state.metrics
-    else:
-        metrics_ = {}
-
-with col2:
-    #st.text_area("Metrics:", value=metrics, height=100, disabled=True)
-    if len(metrics_) > 0: 
-        st.json(metrics_)
+        for q in st.session_state.recent_questions:
+            if q["question"] == st.session_state.last_question:
+                q["metrics"] = {"metrics": st.session_state.metrics}
 
-save_recent_question(question, st.session_state.metrics)
+        # Save updated data to file
+        save_recent_questions(st.session_state.recent_questions)
 
+with col2:
+    st.text_area("Metrics:", value=st.session_state.metrics, height=100, disabled=True)
 
+st.experimental_rerun()

data_processing.py

@@ -23,10 +23,10 @@ query_dataset_data = {}
 
 RECENT_QUESTIONS_FILE = "data_local/recent_questions.json"
 
-# Ensure the file exists and initialize if empty
-if not os.path.exists(RECENT_QUESTIONS_FILE):
-    with open(RECENT_QUESTIONS_FILE, "w") as file:
-        json.dump({"questions": []}, file, indent=4)
+# # Ensure the file exists and initialize if empty
+# if not os.path.exists(RECENT_QUESTIONS_FILE):
+#     with open(RECENT_QUESTIONS_FILE, "w") as file:
+#         json.dump({"questions": []}, file, indent=4)
 
 all_documents = []
 ragbench = {}
@@ -130,27 +130,36 @@ def rerank_documents(query, retrieved_docs):
     ranked_docs = [doc for _, doc in sorted(zip(scores, retrieved_docs), reverse=True)]
     return ranked_docs[:5]  # Return top 5 most relevant
 
+# def load_recent_questions():
+#     if os.path.exists(RECENT_QUESTIONS_FILE):
+#         with open(RECENT_QUESTIONS_FILE, "r") as file:
+#             return json.load(file)
+#     return {"questions": []}  
+
+# def save_recent_question(question, metrics_1):
+#     data = load_recent_questions()
+
+#     # Append new question & metrics
+#     data["questions"].append({
+#         "question": question,
+#         "metrics": metrics_1
+#     })
+
+#     # # Keep only the last 5 questions
+#     # data["questions"]  = data["questions"][-5:]
+
+#     # Write back to file
+#     with open(RECENT_QUESTIONS_FILE, "w") as file:
+#         json.dump(data, file, indent=4)
+
+   # Load previous questions from file
 def load_recent_questions():
     if os.path.exists(RECENT_QUESTIONS_FILE):
         with open(RECENT_QUESTIONS_FILE, "r") as file:
             return json.load(file)
-    return {"questions": []}  
-
-def save_recent_question(question, metrics):
-    data = load_recent_questions()
-
-    if "question" in data["questions"] and question not in data["questions"]["question"]:
-        # Append new question & metrics
-        data["questions"].append({
-            "question": question,
-            "metrics": metrics
-        })
+    return []
 
-    # Keep only the last 5 questions
-    data["questions"]  = data["questions"][-5:]
-
-    # Write back to file
+# Save questions to file
+def save_recent_questions(data):
     with open(RECENT_QUESTIONS_FILE, "w") as file:
-        json.dump(data, file, indent=4)
-
-   
+        json.dump(data, file, indent=4)

evaluation.py

@@ -85,7 +85,7 @@ def adherence(response, relevant_documents):
     response_tokens = set(response.split())
     relevant_tokens = set(combined_docs.split())
     supported_tokens = response_tokens.intersection(relevant_tokens)
-    return len(supported_tokens) / len(response_tokens)
+    return len(supported_tokens) / len(response_tokens) >= 0.5
 
 # Step 6: Compute RMSE for metrics
 def compute_rmse(predicted_values, ground_truth_values):
@@ -93,7 +93,7 @@ def compute_rmse(predicted_values, ground_truth_values):
 
 def calculate_metrics(question, q_dataset, response, docs, time_taken):
     data = load_query_dataset(q_dataset)
-    ground_truth_answer = retrieve_ground_truths(question, data)  # Store the ground truth answer
+    ground_truth_answer, ground_truth_metrics = retrieve_ground_truths(question, data)  # Store the ground truth answer
 
     # Ensure ground_truth_answer is not empty before proceeding
     if ground_truth_answer is None:
@@ -104,12 +104,20 @@ def calculate_metrics(question, q_dataset, response, docs, time_taken):
         "context_relevance": context_relevance(question, docs),
         "context_utilization": context_utilization(response, docs),
         "completeness": completeness(response, ground_truth_answer),
-        "adherence": adherence(response, docs),
+        "adherence": adherence(response, docs),  
+    }
+
+    rmse = compute_rmse(predicted_metrics, ground_truth_metrics),
+
+    metrics = {
+        "RMSE": rmse,
+        "metrics":predicted_metrics,
         "response_time": time_taken,
         "ground_truth": ground_truth_answer,
-        "RAG_model_response": response
+        "RAG_model_response": response, 
     }
-    return predicted_metrics
+
+    return metrics
 
 def retrieve_ground_truths(question, dataset):
     for split_name, instances in dataset.items():
@@ -118,18 +126,18 @@ def retrieve_ground_truths(question, dataset):
             #if instance['question'] == question:
             if is_similar(instance['question'], question):
                 instance_id = instance['id']
-                instance_response = instance['response']
-                # ground_truth_metrics = {
-                #     "context_relevance": instance['relevance_score'],
-                #     "context_utilization": instance['utilization_score'],
-                #     "completeness": instance['completeness_score'],
-                #     "adherence": instance['adherence_score']
-                # }
+                ground_truth = instance['response']
+                ground_truth_metrics_ = {
+                    "context_relevance": instance['relevance_score'],
+                    "context_utilization": instance['utilization_score'],
+                    "completeness": instance['completeness_score'],
+                    "adherence": instance['adherence_score']
+                }
                 print(f"Match found in {split_name} split!")
-                print(f"ID: {instance_id}, Response: {instance_response}")
-                return instance_response  # Return ground truth response immediately
+                print(f"ID: {instance_id}, Response: {ground_truth}")
+                return ground_truth , ground_truth_metrics_  # Return ground truth response immediately
 
-    return None  
+    return None, None
 
 def is_similar(question1, question2, threshold=0.85):
     vectorizer = TfidfVectorizer()