Legend + FAQ

README.md

@@ -10,4 +10,24 @@ pinned: false
 short_description: A dashboard
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# TCID
+
+This space displays the state of the `transformers` CI on two hardwares, for a subset of models. The CI is run daily, on both AMD MI325 and Nvidia A10. The CI runs a different number of tests for each model. When a test finishes, it is assigned a status depending on its outcome:
+
+- passed: the test finsihed and the expected output (or outputs) were retrieved;
+- failed: the test either did not finish or the output was different from the expected output;
+- skipped: the test was not run, which usually happens when a test is incompatible with a model. For instance, some models skip `flash-attention`-related tests because they are incompatible with `flash-attention`;
+- error: the test did not finish and python crashed;
+
+The dashboard is divided in two main parts:
+
+## Summary page
+
+On the summary page, you can see a snapshot of the mix of test passed, failed and skipped for each model. The summary page also features an "Overall failures rate" for AMD and NVIDIA, which is computed this way:
+```overall_failure_rate = (failed + error) / (passed + failed + error)``` 
+
+We do not account for the test skipped in this overall failure rate, because skipped test have no chance to neither pass nor fail.
+
+## Models page
+
+From the sidebar, you can access a detailled view of each model. In it, you will find the breakdown of test statuses and the names of the test that failed for single and multi-gpu runs.

app.py

@@ -229,7 +229,10 @@ with gr.Blocks(title="Model Test Results Dashboard", css=load_css()) as demo:
                 if amd_multi_link and amd_single_link and nvidia_multi_link and nvidia_single_link:
                     break
 
-            links_md = "🔗 **CI Jobs:**\n\n"
+
+            # Add FAQ link at the bottom
+            links_md = "❓ [**FAQ**](https://huggingface.co/spaces/ror/tcid/blob/main/README.md)\n\n"
+            links_md += "🔗 **CI Jobs:**\n\n"
 
             # AMD links
             if amd_multi_link or amd_single_link:
@@ -254,7 +257,7 @@ with gr.Blocks(title="Model Test Results Dashboard", css=load_css()) as demo:
             return links_md
         except Exception as e:
             logger.error(f"getting CI links: {e}")
-            return "🔗 **CI Jobs:** *Error loading links*"
+            return "🔗 **CI Jobs:** *Error loading links*\n\n❓ **[FAQ](README.md)**"
 
 
     # Auto-update CI links when the interface loads

styles.css

@@ -1,4 +1,8 @@
-/* Global dark theme */
+/* Global dark theme with configurable bottom margin */
+:root {
+    --main-content-bottom-margin: 10px; /* Configurable bottom margin for main content */
+}
+
 .gradio-container {
     background-color: #000000 !important;
     color: white !important;
@@ -500,7 +504,7 @@ h1, h2, h3, p, .markdown {
 /* Main content area */
 .main-content {
     background-color: #000000 !important;
-    padding: 0px 20px 40px 20px !important;
+    padding: 0px 20px var(--main-content-bottom-margin, 10px) 20px !important;
     margin-left: 300px !important;
     height: 100vh !important;
     overflow-y: auto !important;

summary_page.py

@@ -11,10 +11,10 @@ BAR_WIDTH = COLUMN_WIDTH * 0.8  # 80% of column width for bars
 BAR_MARGIN = COLUMN_WIDTH * 0.1  # 10% margin on each side
 
 # Figure dimensions
-FIGURE_WIDTH = 20  # Wider to accommodate columns
-MAX_HEIGHT = 12  # Maximum height in inches
-MIN_HEIGHT_PER_ROW = 2.2
-FIGURE_PADDING = 2
+FIGURE_WIDTH = 22  # Wider to accommodate columns and legend
+MAX_HEIGHT = 14  # Maximum height in inches
+MIN_HEIGHT_PER_ROW = 2.8
+FIGURE_PADDING = 1
 
 # Bar styling
 BAR_HEIGHT_RATIO = 0.22  # Bar height as ratio of vertical spacing
@@ -56,13 +56,13 @@ def calculate_overall_failure_rates(df: pd.DataFrame, available_models: list[str
         amd_stats, nvidia_stats = extract_model_data(row)[:2]
         
         # AMD totals
-        amd_total = sum(amd_stats.values())
+        amd_total = amd_stats['passed'] + amd_stats['failed'] + amd_stats['error']
         if amd_total > 0:
             total_amd_tests += amd_total
             total_amd_failures += amd_stats['failed'] + amd_stats['error']
         
         # NVIDIA totals
-        nvidia_total = sum(nvidia_stats.values())
+        nvidia_total = nvidia_stats['passed'] + nvidia_stats['failed'] + nvidia_stats['error']
         if nvidia_total > 0:
             total_nvidia_tests += nvidia_total
             total_nvidia_failures += nvidia_stats['failed'] + nvidia_stats['error']
@@ -180,9 +180,42 @@ def create_summary_page(df: pd.DataFrame, available_models: list[str]) -> plt.Fi
         # Increment counter for next visible model
         visible_model_count += 1
 
+
+    # Add legend horizontally in bottom right corner
+    patch_height = 0.3
+    patch_width = 3
+
+    legend_start_x = 68.7
+    legend_y = max_y + 1
+    legend_spacing = 10
+    legend_font_size = 15
+
+    # Add failure rate explanation text on the left
+    # explanation_text = "Failure rate = failed / (passed + failed)"
+    # ax.text(0, legend_y, explanation_text, 
+    #        ha='left', va='bottom', color='#CCCCCC', 
+    #        fontsize=legend_font_size, fontfamily='monospace', style='italic')
+    
+    # Legend entries
+    legend_items = [
+        ('passed', 'Passed'),
+        ('failed', 'Failed'), 
+        ('skipped', 'Skipped'),
+    ]
+    
+    for i, (status, label) in enumerate(legend_items):
+        x_pos = legend_start_x + i * legend_spacing
+        # Small colored square
+        ax.add_patch(plt.Rectangle((x_pos - 0.6, legend_y), patch_width, -patch_height,
+                                 facecolor=COLORS[status], alpha=0.9))
+        # Status label
+        ax.text(x_pos + patch_width, legend_y, label, 
+               ha='left', va='bottom', color='#CCCCCC', 
+               fontsize=legend_font_size, fontfamily='monospace')
+
     # Style the axes to be completely invisible and span full width
     ax.set_xlim(-5, 105)  # Slightly wider to accommodate labels
-    ax.set_ylim(0, max_y)
+    ax.set_ylim(0, max_y + 1)  # Add some padding at the top for title
     ax.set_xlabel('')
     ax.set_ylabel('')
     ax.spines['bottom'].set_visible(False)