feat: extract spans and HF_TOKEN import

app.py

@@ -7,6 +7,7 @@ from pathlib import Path
 import gradio as gr
 from leptonai import Client
 
+HF_TOKEN = os.environ.get("HF_TOKEN", None)
 LEPTON_API_TOKEN = os.environ.get("LEPTON_API_TOKEN", None)
 
 client = Client("https://yb15a7dy-glider.tin.lepton.run", "glider", LEPTON_API_TOKEN)
@@ -120,6 +121,24 @@ def format_string(retrieved_context, user_input, model_output, gold_answer):
         parts.append(f"<GOLD ANSWER>\n{gold_answer}\n</GOLD ANSWER>")
     return "\n".join(parts)
 
+def extract_spans(input_string):
+    # Regex patterns to extract content within the reasoning, highlight, and score tags
+    reasoning_pattern = r"<reasoning>\s*(.*?)\s*</reasoning>"
+    highlight_pattern = r"<highlight>\s*(.*?)\s*</highlight>"
+    score_pattern = r"<score>\s*(\d+)\s*</score(?:\>|)"
+    
+    # Using re.search to extract the contents based on our defined patterns
+    reasoning_match = re.search(reasoning_pattern, input_string, re.DOTALL)
+    highlight_match = re.search(highlight_pattern, input_string)
+    score_match = re.search(score_pattern, input_string)
+    
+    # Extracting the matched groups if present
+    reasoning = reasoning_match.group(1) if reasoning_match else None
+    highlight = highlight_match.group(1).strip() if highlight_match else None
+    score = int(score_match.group(1)) if score_match else None
+    # Return a dictionary with the extracted content
+    return score, reasoning, highlight
+
 def model_call(model_output, user_input, gold_answer, retrieved_context, pass_criteria, rubric):
     if model_output == "" or user_input == "" or pass_criteria == "":
         return "", "", ""