RETRY

src/about.py

@@ -1,16 +1,9 @@
 from dataclasses import dataclass
 from enum import Enum
 
-@dataclass
-class Task:
-    benchmark: str
-    metric: str
-    col_name: str
-
-# Select your tasks here
+# NO TASKS - ONLY P-VALUES
 # ---------------------------------------------------
 class Tasks(Enum):
-    # No tasks - we only care about p-values
     pass
 
 NUM_FEWSHOT = 0 # Not used
@@ -21,8 +14,7 @@ TITLE = """<h1 align="center" id="space-title">Model Tracing Leaderboard</h1>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-This leaderboard evaluates specific language models based on their perplexity scores and 
-structural similarity to Llama-2-7B using model tracing analysis.
+This leaderboard evaluates specific language models based on their structural similarity to Llama-2-7B using model tracing analysis.
 
 **Models Evaluated:**
 - `lmsys/vicuna-7b-v1.5` - Vicuna 7B v1.5

src/leaderboard/read_evals.py

@@ -75,23 +75,14 @@ class EvalResult:
         )
 
     def to_dict(self):
-        """Converts the Eval Result to a dict compatible with our dataframe display"""
+        """Converts the Eval Result to a dict compatible with our dataframe display - P-VALUES ONLY"""
         import sys
         
-        sys.stderr.write(f"\n=== PROCESSING RESULT TO_DICT ===\n")
+        sys.stderr.write(f"\n=== PROCESSING RESULT TO_DICT (P-VALUES ONLY) ===\n")
         sys.stderr.write(f"Processing result for model: {self.full_model}\n")
-        sys.stderr.write(f"Raw results: {self.results}\n")
-        sys.stderr.write(f"Model precision: {self.precision}\n")
-        sys.stderr.write(f"Model type: {self.model_type}\n")
-        sys.stderr.write(f"Weight type: {self.weight_type}\n")
         sys.stderr.flush()
         
-        # No task-based scoring - we only care about p-values
-        average = 0  # Default average since we don't have tasks
-        sys.stderr.write(f"No task-based scoring, using default average: {average}\n")
-        sys.stderr.flush()
-        
-        # Create data dictionary with comprehensive debugging
+        # Create data dictionary - NO TASK PROCESSING AT ALL
         data_dict = {}
         
         # Add core columns
@@ -103,7 +94,6 @@ class EvalResult:
         data_dict[AutoEvalColumn.architecture.name] = self.architecture
         data_dict[AutoEvalColumn.model.name] = make_clickable_model(self.full_model)
         data_dict[AutoEvalColumn.revision.name] = self.revision
-        data_dict[AutoEvalColumn.average.name] = average
         data_dict[AutoEvalColumn.still_on_hub.name] = self.still_on_hub
         
         # Add default values for missing model info
@@ -112,9 +102,7 @@ class EvalResult:
         data_dict[AutoEvalColumn.likes.name] = 0
         
         # Compute model trace p-value
-        sys.stderr.write(f"\n🧬 COMPUTING MODEL TRACE P-VALUE FOR: {self.full_model}\n")
-        sys.stderr.write(f"   - Revision: {self.revision if self.revision else 'main'}\n")
-        sys.stderr.write(f"   - Precision: {self.precision.value.name.lower()}\n")
+        sys.stderr.write(f"🧬 COMPUTING MODEL TRACE P-VALUE FOR: {self.full_model}\n")
         sys.stderr.flush()
         
         try:
@@ -125,29 +113,17 @@ class EvalResult:
             )
             
             if model_trace_p_value is not None:
-                sys.stderr.write(f"✅ Model trace p-value computed successfully: {model_trace_p_value}\n")
+                sys.stderr.write(f"✅ P-value: {model_trace_p_value}\n")
             else:
-                sys.stderr.write(f"⚠️  Model trace p-value is None (computation failed or not available)\n")
+                sys.stderr.write(f"⚠️ P-value computation failed\n")
                 
         except Exception as e:
-            sys.stderr.write(f"💥 Exception during model trace p-value computation: {e}\n")
-            import traceback
-            sys.stderr.write(f"Traceback: {traceback.format_exc()}\n")
+            sys.stderr.write(f"💥 Exception during p-value computation: {e}\n")
             model_trace_p_value = None
             
         data_dict[AutoEvalColumn.model_trace_p_value.name] = model_trace_p_value
-        sys.stderr.write(f"📝 Added to data_dict: {AutoEvalColumn.model_trace_p_value.name} = {model_trace_p_value}\n")
-        sys.stderr.flush()
         
-        sys.stderr.write(f"Created base data_dict with {len(data_dict)} columns\n")
-        sys.stderr.flush()
-
-        # No task-specific scores - we only have p-values
-        sys.stderr.write("No task-specific scores to add\n")
-        sys.stderr.flush()
-
-        sys.stderr.write(f"Final data dict has {len(data_dict)} columns: {list(data_dict.keys())}\n")
-        sys.stderr.write(f"=== END PROCESSING RESULT TO_DICT ===\n")
+        sys.stderr.write(f"=== END PROCESSING - ONLY P-VALUES ===\n")
         sys.stderr.flush()
         
         return data_dict

src/populate.py

@@ -37,13 +37,13 @@ def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> p
                     sys.stderr.write(f"WARNING: Result for {v.full_model} missing columns: {missing_cols}\n")
                     # Add missing columns with default values
                     for col in missing_cols:
-                        if col in benchmark_cols or col == AutoEvalColumn.average.name:
+                        if col in benchmark_cols:
                             data_dict[col] = 0.0
                         elif col == AutoEvalColumn.model_type_symbol.name:
                             data_dict[col] = "?"
                         else:
                             data_dict[col] = ""
-                    sys.stderr.flush()
+                        sys.stderr.flush()
                 
                 all_data_json.append(data_dict)
                 sys.stderr.write(f"Successfully processed result {i+1}/{len(raw_data)}: {v.full_model}\n")
@@ -79,14 +79,11 @@ def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> p
             return create_empty_dataframe(cols, benchmark_cols)
 
         try:
-            if AutoEvalColumn.average.name in df.columns:
-                df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-                sys.stderr.write("\nSorted DataFrame by average\n")
-            else:
-                sys.stderr.write(f"\nWARNING: Cannot sort by {AutoEvalColumn.average.name} - column not found\n")
+            # No sorting needed - we only have p-values
+            sys.stderr.write("\nNo sorting applied - only p-values\n")
             sys.stderr.flush()
         except Exception as e:
-            sys.stderr.write(f"\nError sorting DataFrame: {e}\n")
+            sys.stderr.write(f"\nError with DataFrame: {e}\n")
             sys.stderr.write("Available columns: " + str(df.columns.tolist()) + "\n")
             sys.stderr.flush()
 
@@ -111,16 +108,10 @@ def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> p
             sys.stderr.flush()
             return create_empty_dataframe(cols, benchmark_cols)
 
-        try:
-            # filter out if perplexity hasn't been evaluated
-            df = df[has_no_nan_values(df, benchmark_cols)]
-            sys.stderr.write("\nFinal DataFrame shape after filtering: " + str(df.shape) + "\n")
-            sys.stderr.write("Final columns: " + str(df.columns.tolist()) + "\n")
-            sys.stderr.flush()
-        except Exception as e:
-            sys.stderr.write(f"Error filtering DataFrame: {e}\n")
-            sys.stderr.flush()
-            # Don't return empty, return the unfiltered DataFrame
+        # No filtering needed - we only have p-values
+        sys.stderr.write("\nFinal DataFrame shape (no filtering): " + str(df.shape) + "\n")
+        sys.stderr.write("Final columns: " + str(df.columns.tolist()) + "\n")
+        sys.stderr.flush()
         
         # Final validation
         if df is None or df.empty:
@@ -150,7 +141,7 @@ def create_empty_dataframe(cols: list, benchmark_cols: list) -> pd.DataFrame:
     empty_df = pd.DataFrame(columns=cols)
     # Ensure correct column types
     for col in cols:
-        if col in benchmark_cols or col == AutoEvalColumn.average.name:
+        if col in benchmark_cols:
             empty_df[col] = pd.Series(dtype=float)
         else:
             empty_df[col] = pd.Series(dtype=str)