Standardize loggin

climateqa/logging.py

@@ -4,13 +4,38 @@ import json
 from huggingface_hub import HfApi
 import gradio as gr
 import csv
+import pandas as pd
+import io
+from typing import TypedDict, List
+from constants import DOCUMENT_METADATA_DEFAULT_VALUES
+
 
 def serialize_docs(docs:list)->list:
+    """Convert document objects to a simplified format compatible with Hugging Face datasets.
+    
+    This function processes document objects by extracting their page content and metadata,
+    normalizing the metadata structure to ensure consistency. It applies default values 
+    from DOCUMENT_METADATA_DEFAULT_VALUES for any missing metadata fields.
+    
+    Args:
+        docs (list): List of document objects, each with page_content and metadata attributes
+        
+    Returns:
+        list: List of dictionaries with standardized "page_content" and "metadata" fields
+    """
     new_docs = []
     for doc in docs:
         new_doc = {}
         new_doc["page_content"] = doc.page_content
-        new_doc["metadata"] = doc.metadata
+        new_doc["metadata"] = {}
+        
+        # Define default empty values by field type
+
+        
+        # Copy each metadata field, using default if missing
+        for field, default_value in DOCUMENT_METADATA_DEFAULT_VALUES.items():
+            new_doc["metadata"][field] = doc.metadata.get(field, default_value)
+            
         new_docs.append(new_doc)
     return new_docs
 
@@ -108,7 +133,7 @@ def log_on_huggingface(log_filename, logs):
             return
 
         # Get repository name from environment or use default
-        repo_id = os.getenv("HF_DATASET_REPO", "timeki/climateqa_logs")
+        repo_id = os.getenv("HF_DATASET_REPO", "Ekimetrics/climateqa_logs")
         
         # Initialize HfApi
         api = HfApi(token=hf_token)
@@ -159,9 +184,12 @@ def log_interaction_to_huggingface(history, output_query, sources, docs, share_c
             }
             # Log to Hugging Face
             log_on_huggingface(f"chat/{timestamp}.json", logs)
+            print(f"Logged interaction to Hugging Face")
+        else:
+            print("Did not log to Hugging Face because GRADIO_ENV is local")
     except Exception as e:
         print(f"Error logging to Hugging Face: {e}")
-        error_msg = f"ClimateQ&A Error: {str(e)[:100]} - The error has been noted, try another question and if the error remains, you can contact us :)"
+        error_msg = f"ClimateQ&A Error: {str(e)[:100]})"
         raise gr.Error(error_msg)
 
 def log_drias_interaction_to_huggingface(query, sql_query, user_id):