chore: add u-math results

data/u_math_eval_results.json

@@ -1,13 +1,1250 @@
 [
+  {
+    "model_name": "llava-hf/llava-v1.6-mistral-7b-hf",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      3.2727,
+      3.6667,
+      1.5
+    ],
+    "algebra": [
+      0.0722,
+      0.0867,
+      0.0
+    ],
+    "differential_calc": [
+      0.0227,
+      0.0133,
+      0.0429
+    ],
+    "integral_calc": [
+      0.0,
+      0.0,
+      0.0
+    ],
+    "multivariable_calculus": [
+      0.0281,
+      0.0333,
+      0.0
+    ],
+    "precalculus_review": [
+      0.0562,
+      0.06,
+      0.0
+    ],
+    "sequences_series": [
+      0.026,
+      0.0267,
+      0.0
+    ]
+  },
+  {
+    "model_name": "mistralai/Pixtral-12B-2409",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      15.5455,
+      15.5556,
+      15.5
+    ],
+    "algebra": [
+      0.4111,
+      0.4467,
+      0.2333
+    ],
+    "differential_calc": [
+      0.1182,
+      0.0133,
+      0.3429
+    ],
+    "integral_calc": [
+      0.0048,
+      0.0067,
+      0.0
+    ],
+    "multivariable_calculus": [
+      0.0281,
+      0.0333,
+      0.0
+    ],
+    "precalculus_review": [
+      0.3,
+      0.32,
+      0.0
+    ],
+    "sequences_series": [
+      0.1104,
+      0.1133,
+      0.0
+    ]
+  },
+  {
+    "model_name": "meta-llama/Llama-3.2-11B-Vision-Instruct",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      17.0,
+      18.5556,
+      10.0
+    ],
+    "algebra": [
+      0.4667,
+      0.54,
+      0.1
+    ],
+    "differential_calc": [
+      0.0727,
+      0.0133,
+      0.2
+    ],
+    "integral_calc": [
+      0.0144,
+      0.0133,
+      0.0172
+    ],
+    "multivariable_calculus": [
+      0.0449,
+      0.0467,
+      0.0357
+    ],
+    "precalculus_review": [
+      0.4125,
+      0.4333,
+      0.1
+    ],
+    "sequences_series": [
+      0.0649,
+      0.0667,
+      0.0
+    ]
+  },
+  {
+    "model_name": "llava-hf/llava-onevision-qwen2-7b-ov-chat-hf",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      17.7273,
+      20.6667,
+      4.5
+    ],
+    "algebra": [
+      0.5167,
+      0.6067,
+      0.0667
+    ],
+    "differential_calc": [
+      0.0455,
+      0.04,
+      0.0571
+    ],
+    "integral_calc": [
+      0.0144,
+      0.0133,
+      0.0172
+    ],
+    "multivariable_calculus": [
+      0.0506,
+      0.0533,
+      0.0357
+    ],
+    "precalculus_review": [
+      0.4125,
+      0.4333,
+      0.1
+    ],
+    "sequences_series": [
+      0.0909,
+      0.0933,
+      0.0
+    ]
+  },
+  {
+    "model_name": "mistralai/Mathstral-7B-v0.1",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      18.0,
+      20.6667,
+      6.0
+    ],
+    "algebra": [
+      0.4389,
+      0.5133,
+      0.0667
+    ],
+    "differential_calc": [
+      0.0591,
+      0.04,
+      0.1
+    ],
+    "integral_calc": [
+      0.0144,
+      0.0133,
+      0.0172
+    ],
+    "multivariable_calculus": [
+      0.073,
+      0.08,
+      0.0357
+    ],
+    "precalculus_review": [
+      0.4625,
+      0.4867,
+      0.1
+    ],
+    "sequences_series": [
+      0.1039,
+      0.1067,
+      0.0
+    ]
+  },
+  {
+    "model_name": "mistralai/Ministral-8B-Instruct-2410",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      18.2727,
+      21.4444,
+      4.0
+    ],
+    "algebra": [
+      0.5222,
+      0.62,
+      0.0333
+    ],
+    "differential_calc": [
+      0.05,
+      0.0333,
+      0.0857
+    ],
+    "integral_calc": [
+      0.0096,
+      0.0133,
+      0.0
+    ],
+    "multivariable_calculus": [
+      0.0562,
+      0.06,
+      0.0357
+    ],
+    "precalculus_review": [
+      0.4375,
+      0.4667,
+      0.0
+    ],
+    "sequences_series": [
+      0.0909,
+      0.0933,
+      0.0
+    ]
+  },
+  {
+    "model_name": "AI-MO/NuminaMath-7B-CoT",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      19.1818,
+      22.7778,
+      3.0
+    ],
+    "algebra": [
+      0.5222,
+      0.6267,
+      0.0
+    ],
+    "differential_calc": [
+      0.05,
+      0.04,
+      0.0714
+    ],
+    "integral_calc": [
+      0.0096,
+      0.0133,
+      0.0
+    ],
+    "multivariable_calculus": [
+      0.0562,
+      0.06,
+      0.0357
+    ],
+    "precalculus_review": [
+      0.4812,
+      0.5133,
+      0.0
+    ],
+    "sequences_series": [
+      0.1104,
+      0.1133,
+      0.0
+    ]
+  },
+  {
+    "model_name": "Qwen/Qwen2-VL-7B-Instruct",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      20.3636,
+      21.4444,
+      15.5
+    ],
+    "algebra": [
+      0.5389,
+      0.6267,
+      0.1
+    ],
+    "differential_calc": [
+      0.1364,
+      0.0467,
+      0.3286
+    ],
+    "integral_calc": [
+      0.0192,
+      0.0067,
+      0.0517
+    ],
+    "multivariable_calculus": [
+      0.0674,
+      0.0667,
+      0.0714
+    ],
+    "precalculus_review": [
+      0.425,
+      0.4533,
+      0.0
+    ],
+    "sequences_series": [
+      0.0844,
+      0.0867,
+      0.0
+    ]
+  },
+  {
+    "model_name": "meta-llama/Llama-3.1-8B-Instruct",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      22.2727,
+      26.1111,
+      5.0
+    ],
+    "algebra": [
+      0.5,
+      0.5933,
+      0.0333
+    ],
+    "differential_calc": [
+      0.0636,
+      0.0667,
+      0.0571
+    ],
+    "integral_calc": [
+      0.0769,
+      0.0933,
+      0.0345
+    ],
+    "multivariable_calculus": [
+      0.1011,
+      0.1133,
+      0.0357
+    ],
+    "precalculus_review": [
+      0.5188,
+      0.5467,
+      0.1
+    ],
+    "sequences_series": [
+      0.1558,
+      0.1533,
+      0.25
+    ]
+  },
+  {
+    "model_name": "AI-MO/NuminaMath-72B-CoT",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      25.0,
+      29.6667,
+      4.0
+    ],
+    "algebra": [
+      0.6278,
+      0.7467,
+      0.0333
+    ],
+    "differential_calc": [
+      0.0591,
+      0.0667,
+      0.0429
+    ],
+    "integral_calc": [
+      0.0385,
+      0.04,
+      0.0345
+    ],
+    "multivariable_calculus": [
+      0.1011,
+      0.1133,
+      0.0357
+    ],
+    "precalculus_review": [
+      0.5938,
+      0.6267,
+      0.1
+    ],
+    "sequences_series": [
+      0.1818,
+      0.1867,
+      0.0
+    ]
+  },
+  {
+    "model_name": "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      25.9091,
+      30.2222,
+      6.5
+    ],
+    "algebra": [
+      0.6611,
+      0.78,
+      0.0667
+    ],
+    "differential_calc": [
+      0.0591,
+      0.0533,
+      0.0714
+    ],
+    "integral_calc": [
+      0.0673,
+      0.08,
+      0.0345
+    ],
+    "multivariable_calculus": [
+      0.118,
+      0.1333,
+      0.0357
+    ],
+    "precalculus_review": [
+      0.5938,
+      0.6267,
+      0.1
+    ],
+    "sequences_series": [
+      0.1494,
+      0.14,
+      0.5
+    ]
+  },
+  {
+    "model_name": "meta-llama/Llama-3.1-70B-Instruct",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      28.4545,
+      33.6667,
+      5.0
+    ],
+    "algebra": [
+      0.6889,
+      0.82,
+      0.0333
+    ],
+    "differential_calc": [
+      0.0909,
+      0.1067,
+      0.0571
+    ],
+    "integral_calc": [
+      0.0433,
+      0.04,
+      0.0517
+    ],
+    "multivariable_calculus": [
+      0.1236,
+      0.14,
+      0.0357
+    ],
+    "precalculus_review": [
+      0.6,
+      0.64,
+      0.0
+    ],
+    "sequences_series": [
+      0.2727,
+      0.2733,
+      0.25
+    ]
+  },
+  {
+    "model_name": "Qwen/Qwen2-VL-72B-Instruct",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      31.1818,
+      32.2222,
+      26.5
+    ],
+    "algebra": [
+      0.7167,
+      0.8067,
+      0.2667
+    ],
+    "differential_calc": [
+      0.1909,
+      0.0933,
+      0.4
+    ],
+    "integral_calc": [
+      0.0529,
+      0.02,
+      0.1379
+    ],
+    "multivariable_calculus": [
+      0.1685,
+      0.1467,
+      0.2857
+    ],
+    "precalculus_review": [
+      0.6188,
+      0.6533,
+      0.1
+    ],
+    "sequences_series": [
+      0.2078,
+      0.2133,
+      0.0
+    ]
+  },
+  {
+    "model_name": "nvidia/Llama-3.1-Nemotron-70B-Instruct-HF",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      31.3636,
+      37.4444,
+      4.0
+    ],
+    "algebra": [
+      0.7,
+      0.84,
+      0.0
+    ],
+    "differential_calc": [
+      0.1091,
+      0.1467,
+      0.0286
+    ],
+    "integral_calc": [
+      0.0385,
+      0.04,
+      0.0345
+    ],
+    "multivariable_calculus": [
+      0.2247,
+      0.2533,
+      0.0714
+    ],
+    "precalculus_review": [
+      0.6125,
+      0.64,
+      0.2
+    ],
+    "sequences_series": [
+      0.3182,
+      0.3267,
+      0.0
+    ]
+  },
+  {
+    "model_name": "meta-llama/Llama-3.2-90B-Vision-Instruct",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      32.6364,
+      36.3333,
+      16.0
+    ],
+    "algebra": [
+      0.7556,
+      0.8533,
+      0.2667
+    ],
+    "differential_calc": [
+      0.1545,
+      0.1067,
+      0.2571
+    ],
+    "integral_calc": [
+      0.024,
+      0.0267,
+      0.0172
+    ],
+    "multivariable_calculus": [
+      0.2022,
+      0.2267,
+      0.0714
+    ],
+    "precalculus_review": [
+      0.625,
+      0.6533,
+      0.2
+    ],
+    "sequences_series": [
+      0.3117,
+      0.3133,
+      0.25
+    ]
+  },
+  {
+    "model_name": "Qwen/Qwen2.5-7B-Instruct",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      33.8182,
+      40.0,
+      6.0
+    ],
+    "algebra": [
+      0.7333,
+      0.86,
+      0.1
+    ],
+    "differential_calc": [
+      0.0909,
+      0.1267,
+      0.0143
+    ],
+    "integral_calc": [
+      0.1058,
+      0.1,
+      0.1207
+    ],
+    "multivariable_calculus": [
+      0.2303,
+      0.2667,
+      0.0357
+    ],
+    "precalculus_review": [
+      0.7062,
+      0.7533,
+      0.0
+    ],
+    "sequences_series": [
+      0.2857,
+      0.2933,
+      0.0
+    ]
+  },
+  {
+    "model_name": "claude-sonnet-3-5",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      35.0909,
+      36.1111,
+      30.5
+    ],
+    "algebra": [
+      0.6889,
+      0.76,
+      0.3333
+    ],
+    "differential_calc": [
+      0.2136,
+      0.12,
+      0.4143
+    ],
+    "integral_calc": [
+      0.101,
+      0.0733,
+      0.1724
+    ],
+    "multivariable_calculus": [
+      0.2247,
+      0.2133,
+      0.2857
+    ],
+    "precalculus_review": [
+      0.6312,
+      0.6533,
+      0.3
+    ],
+    "sequences_series": [
+      0.3442,
+      0.3467,
+      0.25
+    ]
+  },
   {
     "model_name": "gpt-4o-mini-2024-07-18",
-    "judge_model_name": "gpt-4o-mini-2024-07-18",
-    "u_math": [0.5123, 0.2345, 0.1234],
-    "differential_calc": [0.5123, 0.2345, 0.1234],
-    "integral_calc": [0.43, 0.23, 0.34],
-    "algebra": [0.98, 0.12, 0.34],
-    "multivariable_calculus": [0.98, 0.12, 0.34],
-    "precalculus_review": [0.8412, 0.1234, 0.1234],
-    "sequences_series": [0.1234, 0.1234, 0.1234]
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      37.1818,
+      40.3333,
+      23.0
+    ],
+    "algebra": [
+      0.7611,
+      0.88,
+      0.1667
+    ],
+    "differential_calc": [
+      0.2136,
+      0.1667,
+      0.3143
+    ],
+    "integral_calc": [
+      0.0577,
+      0.04,
+      0.1034
+    ],
+    "multivariable_calculus": [
+      0.2584,
+      0.24,
+      0.3571
+    ],
+    "precalculus_review": [
+      0.7375,
+      0.7733,
+      0.2
+    ],
+    "sequences_series": [
+      0.3182,
+      0.32,
+      0.25
+    ]
+  },
+  {
+    "model_name": "meta-llama/Llama-3.3-70B-Instruct",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      37.2727,
+      43.4444,
+      9.5
+    ],
+    "algebra": [
+      0.7333,
+      0.8733,
+      0.0333
+    ],
+    "differential_calc": [
+      0.1773,
+      0.2,
+      0.1286
+    ],
+    "integral_calc": [
+      0.1154,
+      0.1133,
+      0.1207
+    ],
+    "multivariable_calculus": [
+      0.3315,
+      0.38,
+      0.0714
+    ],
+    "precalculus_review": [
+      0.6312,
+      0.6733,
+      0.0
+    ],
+    "sequences_series": [
+      0.3571,
+      0.3667,
+      0.0
+    ]
+  },
+  {
+    "model_name": "Qwen/Qwen2.5-Math-7B-Instruct",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      38.3636,
+      45.2222,
+      7.5
+    ],
+    "algebra": [
+      0.7389,
+      0.8733,
+      0.0667
+    ],
+    "differential_calc": [
+      0.1455,
+      0.1867,
+      0.0571
+    ],
+    "integral_calc": [
+      0.0865,
+      0.08,
+      0.1034
+    ],
+    "multivariable_calculus": [
+      0.3202,
+      0.36,
+      0.1071
+    ],
+    "precalculus_review": [
+      0.7562,
+      0.8067,
+      0.0
+    ],
+    "sequences_series": [
+      0.3961,
+      0.4067,
+      0.0
+    ]
+  },
+  {
+    "model_name": "mistralai/Pixtral-Large-Instruct-2411",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      39.7273,
+      42.8889,
+      25.5
+    ],
+    "algebra": [
+      0.7722,
+      0.86,
+      0.3333
+    ],
+    "differential_calc": [
+      0.2045,
+      0.1533,
+      0.3143
+    ],
+    "integral_calc": [
+      0.1106,
+      0.0933,
+      0.1552
+    ],
+    "multivariable_calculus": [
+      0.309,
+      0.32,
+      0.25
+    ],
+    "precalculus_review": [
+      0.6938,
+      0.7267,
+      0.2
+    ],
+    "sequences_series": [
+      0.4156,
+      0.42,
+      0.25
+    ]
+  },
+  {
+    "model_name": "mistralai/Mistral-Large-Instruct-2411",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      40.3636,
+      48.1111,
+      5.5
+    ],
+    "algebra": [
+      0.7333,
+      0.8667,
+      0.0667
+    ],
+    "differential_calc": [
+      0.1682,
+      0.2333,
+      0.0286
+    ],
+    "integral_calc": [
+      0.125,
+      0.1533,
+      0.0517
+    ],
+    "multivariable_calculus": [
+      0.3315,
+      0.3733,
+      0.1071
+    ],
+    "precalculus_review": [
+      0.7562,
+      0.8067,
+      0.0
+    ],
+    "sequences_series": [
+      0.4481,
+      0.4533,
+      0.25
+    ]
+  },
+  {
+    "model_name": "Qwen/Qwen2.5-72B-Instruct",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      41.0,
+      48.5556,
+      7.0
+    ],
+    "algebra": [
+      0.75,
+      0.8867,
+      0.0667
+    ],
+    "differential_calc": [
+      0.1682,
+      0.2267,
+      0.0429
+    ],
+    "integral_calc": [
+      0.1058,
+      0.12,
+      0.069
+    ],
+    "multivariable_calculus": [
+      0.3652,
+      0.4,
+      0.1786
+    ],
+    "precalculus_review": [
+      0.7812,
+      0.8333,
+      0.0
+    ],
+    "sequences_series": [
+      0.4351,
+      0.4467,
+      0.0
+    ]
+  },
+  {
+    "model_name": "gpt-4o-2024-05-13",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      43.3636,
+      45.7778,
+      32.5
+    ],
+    "algebra": [
+      0.7778,
+      0.8733,
+      0.3
+    ],
+    "differential_calc": [
+      0.2727,
+      0.2067,
+      0.4143
+    ],
+    "integral_calc": [
+      0.1154,
+      0.08,
+      0.2069
+    ],
+    "multivariable_calculus": [
+      0.3876,
+      0.3867,
+      0.3929
+    ],
+    "precalculus_review": [
+      0.775,
+      0.8067,
+      0.3
+    ],
+    "sequences_series": [
+      0.3896,
+      0.3933,
+      0.25
+    ]
+  },
+  {
+    "model_name": "gpt-4o-2024-08-06",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      43.4545,
+      46.4444,
+      30.0
+    ],
+    "algebra": [
+      0.8111,
+      0.9133,
+      0.3
+    ],
+    "differential_calc": [
+      0.2318,
+      0.1867,
+      0.3286
+    ],
+    "integral_calc": [
+      0.1298,
+      0.1,
+      0.2069
+    ],
+    "multivariable_calculus": [
+      0.4157,
+      0.4133,
+      0.4286
+    ],
+    "precalculus_review": [
+      0.7625,
+      0.7933,
+      0.3
+    ],
+    "sequences_series": [
+      0.3766,
+      0.38,
+      0.25
+    ]
+  },
+  {
+    "model_name": "Qwen/Qwen2.5-32B-Instruct",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      43.8182,
+      51.4444,
+      9.5
+    ],
+    "algebra": [
+      0.7778,
+      0.9267,
+      0.0333
+    ],
+    "differential_calc": [
+      0.2273,
+      0.3,
+      0.0714
+    ],
+    "integral_calc": [
+      0.1202,
+      0.12,
+      0.1207
+    ],
+    "multivariable_calculus": [
+      0.4101,
+      0.4533,
+      0.1786
+    ],
+    "precalculus_review": [
+      0.7875,
+      0.8333,
+      0.1
+    ],
+    "sequences_series": [
+      0.4416,
+      0.4533,
+      0.0
+    ]
+  },
+  {
+    "model_name": "Nexusflow/Athene-V2-Chat",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      46.1818,
+      54.5556,
+      8.5
+    ],
+    "algebra": [
+      0.7444,
+      0.8867,
+      0.0333
+    ],
+    "differential_calc": [
+      0.2455,
+      0.34,
+      0.0429
+    ],
+    "integral_calc": [
+      0.1346,
+      0.16,
+      0.069
+    ],
+    "multivariable_calculus": [
+      0.4607,
+      0.5067,
+      0.2143
+    ],
+    "precalculus_review": [
+      0.8375,
+      0.8867,
+      0.1
+    ],
+    "sequences_series": [
+      0.4935,
+      0.4933,
+      0.5
+    ]
+  },
+  {
+    "model_name": "Qwen/Qwen2.5-Math-72B-Instruct",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      50.1818,
+      59.0,
+      10.5
+    ],
+    "algebra": [
+      0.7833,
+      0.9267,
+      0.0667
+    ],
+    "differential_calc": [
+      0.2636,
+      0.3533,
+      0.0714
+    ],
+    "integral_calc": [
+      0.1971,
+      0.2067,
+      0.1724
+    ],
+    "multivariable_calculus": [
+      0.5,
+      0.58,
+      0.0714
+    ],
+    "precalculus_review": [
+      0.8438,
+      0.9,
+      0.0
+    ],
+    "sequences_series": [
+      0.5714,
+      0.5733,
+      0.5
+    ]
+  },
+  {
+    "model_name": "Qwen/QVQ-72B-Preview",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      50.5455,
+      59.3333,
+      11.0
+    ],
+    "algebra": [
+      0.7833,
+      0.9267,
+      0.0667
+    ],
+    "differential_calc": [
+      0.3182,
+      0.4467,
+      0.0429
+    ],
+    "integral_calc": [
+      0.1731,
+      0.1933,
+      0.1207
+    ],
+    "multivariable_calculus": [
+      0.4888,
+      0.5333,
+      0.25
+    ],
+    "precalculus_review": [
+      0.8688,
+      0.9133,
+      0.2
+    ],
+    "sequences_series": [
+      0.539,
+      0.5467,
+      0.25
+    ]
+  },
+  {
+    "model_name": "google/gemini-1.5-flash",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      51.2727,
+      53.7778,
+      40.0
+    ],
+    "algebra": [
+      0.8444,
+      0.9133,
+      0.5
+    ],
+    "differential_calc": [
+      0.3909,
+      0.36,
+      0.4571
+    ],
+    "integral_calc": [
+      0.1683,
+      0.14,
+      0.2414
+    ],
+    "multivariable_calculus": [
+      0.4494,
+      0.44,
+      0.5
+    ],
+    "precalculus_review": [
+      0.775,
+      0.8067,
+      0.3
+    ],
+    "sequences_series": [
+      0.5649,
+      0.5667,
+      0.5
+    ]
+  },
+  {
+    "model_name": "google/gemini-1.5-pro",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      60.0909,
+      63.4444,
+      45.0
+    ],
+    "algebra": [
+      0.8611,
+      0.9133,
+      0.6
+    ],
+    "differential_calc": [
+      0.4955,
+      0.5067,
+      0.4714
+    ],
+    "integral_calc": [
+      0.2644,
+      0.2733,
+      0.2414
+    ],
+    "multivariable_calculus": [
+      0.6011,
+      0.6067,
+      0.5714
+    ],
+    "precalculus_review": [
+      0.8625,
+      0.8733,
+      0.7
+    ],
+    "sequences_series": [
+      0.6299,
+      0.6333,
+      0.5
+    ]
+  },
+  {
+    "model_name": "Qwen/QwQ-32B-Preview",
+    "judge_model_name": "gpt-4o-2024-08-06",
+    "u_math": [
+      61.4545,
+      71.7778,
+      15.0
+    ],
+    "algebra": [
+      0.7889,
+      0.94,
+      0.0333
+    ],
+    "differential_calc": [
+      0.4364,
+      0.6067,
+      0.0714
+    ],
+    "integral_calc": [
+      0.351,
+      0.3933,
+      0.2414
+    ],
+    "multivariable_calculus": [
+      0.5899,
+      0.6533,
+      0.25
+    ],
+    "precalculus_review": [
+      0.875,
+      0.9267,
+      0.1
+    ],
+    "sequences_series": [
+      0.7792,
+      0.7867,
+      0.5
+    ]
   }
-]
+]

src/populate.py

@@ -30,8 +30,10 @@ def model_size_to_symbol(model_size_in_b_params: int | None) -> str:
         return "🚗"
     elif model_size_in_b_params < 100:
         return "🚚"
-    else:
+    elif model_size_in_b_params < 1000:
         return "🚀"
+    else:
+        return "❓"
 
 
 def model_type_to_symbol(model_type: str) -> str:
@@ -47,6 +49,10 @@ def model_type_to_symbol(model_type: str) -> str:
 def get_hf_data_by_model_name(model_name: str) -> dict:
     """Get model data from Hugging Face API by model name"""
     still_on_hub, _, model_config = is_model_on_hub(model_name, "main", trust_remote_code=True)
+    if not still_on_hub and '/' in model_name:
+        print(f"Model {model_name} is not on the hub, try unsloth/...")
+        model_name = "unsloth/" + model_name.split("/")[-1]
+        still_on_hub, _, model_config = is_model_on_hub(model_name, "main", trust_remote_code=True)
 
     architecture = "Unknown"
     if model_config is not None:
@@ -58,9 +64,11 @@ def get_hf_data_by_model_name(model_name: str) -> dict:
     if still_on_hub:
         info = model_info(repo_id=model_name)
         try:
-            num_params = round(info.safetensors["total"] / 1e9, 3)
-        except AttributeError as e:
+            num_params = round(info.safetensors["total"] / 1e9, 1)
+        except Exception as e:
             print("SafeTensors not found in", model_name, e)
+            if 'Pixtral-12B' in model_name:
+                num_params = 12
             pass
         print("num_params", model_name, num_params)