Update results

results/aggregated_scores.csv

@@ -1,7 +1,7 @@
 Model,Agg S2R,Agg MC,Agg VerilogEval S2R,Agg VerilogEval MC,Agg RTLLM,Agg VeriGen
 DeepSeek R1,75.53,72.96,77.67,77.55,68.49,57.82
 Llama 3.1 405B,53.23,53.88,56.55,54.35,42.26,52.35
-Qwen3 236B A22B,69.82,61.71,74.83,68.36,53.31,39.8
+Qwen3 236B A22B,69.16,63.42,74.83,68.36,50.48,47.15
 Llama 3.(1-3) 70B,39.48,43.29,39.47,40.83,39.53,51.42
 Qwen2.5 72B,49.36,47.23,50.22,50.74,46.51,35.65
 QwQ 32B,62.6,39.46,65.02,38.68,54.6,42.03

results/compute_agg_results.py

@@ -0,0 +1,95 @@
+import csv
+
+
+def agg_S2R_metrics(verilog_eval_rtl, rtllm):
+    if not verilog_eval_rtl or not rtllm:
+        return None
+    w1 = 155
+    w2 = 47
+    result = (w1 * verilog_eval_rtl + w2 * rtllm) / (w1 + w2)
+    return round(result, 2)
+
+
+def agg_MC_metrics(verilog_eval_cc, verigen):
+    if not verilog_eval_cc or not verigen:
+        return None
+    w1 = 155
+    w2 = 17
+    result = (w1 * verilog_eval_cc + w2 * verigen) / (w1 + w2)
+    return round(result, 2)
+
+
+def avg_ppa(metric):
+    result = (
+        float(metric["Power"]) + float(metric["Performance"]) + float(metric["Area"])
+    ) / 3
+    return round(result, 2)
+
+
+#######################################
+
+RESULTS_CSV = "results.csv"
+AGG_CSV = "aggregated_scores.csv"
+
+agg_csv = []
+first_row_agg_csv = [
+    "Model",
+    "Agg S2R",
+    "Agg MC",
+    "Agg VerilogEval S2R",
+    "Agg VerilogEval MC",
+    "Agg RTLLM",
+    "Agg VeriGen",
+]
+agg_csv.append(first_row_agg_csv)
+
+with open(RESULTS_CSV, newline="", encoding="utf-8") as csvfile:
+    reader = csv.reader(csvfile)
+    goals_row = next(reader)[1:]  # Skip the first column (model name)
+    benchmarks_row = next(reader)[1:]
+
+    for model_metrics in reader:
+        model = model_metrics[0]
+        # Create a dict of dicts with keys benchmark, goal
+        results = {}
+        for i, metric in enumerate(model_metrics[1:]):
+            benchmark = benchmarks_row[i]
+            goal = goals_row[i]
+
+            if benchmark not in results:
+                results[benchmark] = {}
+
+            results[benchmark][goal] = metric
+
+        # Aggregate metrics
+        agg_csv_row = []
+        # ['Model','Agg S2R','Agg MC','Agg VerilogEval S2R','Agg VerilogEval MC','Agg RTLLM','Agg VeriGen']
+        for col in first_row_agg_csv:
+            if col == "Model":
+                agg_csv_row.append(model)
+            elif col == "Agg S2R":
+                agg_csv_row.append(
+                    agg_S2R_metrics(
+                        avg_ppa(results["VerilogEval S2R"]), avg_ppa(results["RTLLM"])
+                    )
+                )
+            elif col == "Agg MC":
+                agg_csv_row.append(
+                    agg_S2R_metrics(
+                        avg_ppa(results["VerilogEval MC"]), avg_ppa(results["VeriGen"])
+                    )
+                )
+            elif col == "Agg VerilogEval S2R":
+                agg_csv_row.append(avg_ppa(results["VerilogEval S2R"]))
+            elif col == "Agg VerilogEval MC":
+                agg_csv_row.append(avg_ppa(results["VerilogEval MC"]))
+            elif col == "Agg RTLLM":
+                agg_csv_row.append(avg_ppa(results["RTLLM"]))
+            elif col == "Agg VeriGen":
+                agg_csv_row.append(avg_ppa(results["VeriGen"]))
+
+        agg_csv.append(agg_csv_row)
+
+with open(AGG_CSV, "w", newline="") as csvfile:
+    writer = csv.writer(csvfile)
+    writer.writerows(agg_csv)

results/results.csv

@@ -1,28 +1,28 @@
 ,Syntax (STX),Syntax (STX),Functionality (FNC),Functionality (FNC),Synthesis (SYN),Synthesis (SYN),Power,Power,Performance,Performance,Area,Area,EM,Syntax (STX),Syntax (STX),Functionality (FNC),Functionality (FNC),Synthesis (SYN),Synthesis (SYN),Power,Power,Performance,Performance,Area,Area
 ,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,RTL-Repo,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen
-DeepSeek R1,97.18,89.80,79.74,65.71,79.62,63.27,78.33,71.34,76.49,64.06,78.19,70.08,-1,97.44,96.47,79.49,60.00,79.49,60.00,78.27,50.25,76.43,60.15,77.96,63.07
+DeepSeek R1,97.18,89.80,79.74,65.71,79.62,63.27,78.33,71.34,76.49,64.06,78.19,70.08,-1.00,97.44,96.47,79.49,60.00,79.49,60.00,78.27,50.25,76.43,60.15,77.96,63.07
 Llama 3.1 405B,87.44,77.14,58.97,45.71,58.85,41.63,57.58,50.88,55.93,32.44,56.13,43.45,34.62,88.59,95.29,56.15,52.94,55.90,52.94,55.13,49.22,53.45,52.52,54.48,55.31
-Qwen3 236B A22B,91.28,80.41,76.92,53.06,76.79,51.43,75.25,57.77,73.56,49.20,75.67,52.95,41.94,82.18,91.76,69.62,40.00,69.62,40.00,69.04,39.09,66.89,40.16,69.15,40.14
+Qwen3 236B A22B,91.28,73.88,76.92,51.43,76.79,48.57,75.25,54.61,73.56,46.37,75.67,50.47,41.94,82.18,87.06,69.62,49.41,69.62,49.41,69.04,41.82,66.89,49.64,69.15,49.99
 Llama 3.(1-3) 70B,66.15,73.88,40.64,42.45,40.64,39.18,40.46,40.81,38.08,38.14,39.86,39.65,28.72,84.74,89.41,41.67,51.76,41.67,51.76,41.38,50.61,39.75,51.76,41.36,51.88
 Qwen2.5 72B,82.18,79.59,52.44,45.31,51.92,44.08,51.83,46.47,48.75,45.40,50.09,47.65,37.44,80.90,84.71,52.95,35.29,52.69,35.29,51.66,35.82,49.37,35.20,51.18,35.94
-QwQ 32B,87.95,82.45,66.41,56.73,66.41,52.24,66.15,55.83,63.80,51.91,65.12,56.07,-1,58.97,68.24,40.00,42.35,39.62,42.35,39.40,40.90,37.53,42.31,39.10,42.87
+QwQ 32B,87.95,82.45,66.41,56.73,66.41,52.24,66.15,55.83,63.80,51.91,65.12,56.07,-1.00,58.97,68.24,40.00,42.35,39.62,42.35,39.40,40.90,37.53,42.31,39.10,42.87
 Qwen2.5 32B,88.59,84.08,52.56,50.20,52.18,46.12,52.32,49.73,49.43,46.43,50.82,50.43,28.93,93.21,85.88,41.54,32.94,41.54,32.94,41.31,30.65,40.48,33.11,41.23,32.50
 StarChat2 15B v0.1,88.46,84.90,37.95,44.49,37.95,44.08,37.56,46.95,35.30,43.22,37.19,46.65,13.42,79.74,92.94,36.41,51.76,36.03,51.76,36.08,46.30,34.91,51.49,35.76,52.80
-DeepSeek R1 Distill Qwen 14B,42.18,34.69,25.51,18.37,25.51,16.33,25.36,17.86,24.19,16.48,25.27,17.33,-1,45.00,44.71,25.64,21.18,25.26,21.18,24.79,17.65,23.48,21.08,24.63,21.29
+DeepSeek R1 Distill Qwen 14B,42.18,34.69,25.51,18.37,25.51,16.33,25.36,17.86,24.19,16.48,25.27,17.33,-1.00,45.00,44.71,25.64,21.18,25.26,21.18,24.79,17.65,23.48,21.08,24.63,21.29
 CodeLlama 70B,67.05,69.80,33.08,36.33,33.08,34.29,32.69,37.19,31.46,34.29,32.44,35.95,24.33,90.77,88.24,33.33,35.29,33.33,35.29,33.02,34.03,30.80,35.15,32.99,35.21
 DeepSeek Coder 33B,62.82,83.67,23.33,42.45,23.08,42.04,22.86,42.29,22.81,39.42,22.29,42.71,24.58,75.26,88.24,39.62,31.76,39.36,31.76,38.23,32.16,36.79,31.46,37.90,32.12
 QwenCoder 2.5 32B,87.18,77.96,45.00,43.27,44.87,43.27,44.25,46.82,43.03,43.20,43.76,45.42,31.07,83.72,87.06,45.64,42.35,45.13,42.35,44.59,42.79,43.01,42.24,44.55,43.25
 QwenCoder 2.5 14B,78.97,81.63,37.82,46.12,37.44,45.31,35.94,45.82,34.83,44.64,35.18,46.05,37.53,80.00,83.53,41.67,35.29,41.15,35.29,40.74,34.17,39.20,35.32,40.83,34.67
-DeepCoder 14B,43.85,39.59,28.08,23.67,28.08,22.04,27.94,25.00,26.26,22.00,27.77,23.15,-1,61.92,48.24,34.10,23.53,33.72,23.53,33.70,21.18,32.17,23.43,33.67,23.65
+DeepCoder 14B,43.85,39.59,28.08,23.67,28.08,22.04,27.94,25.00,26.26,22.00,27.77,23.15,-1.00,61.92,48.24,34.10,23.53,33.72,23.53,33.70,21.18,32.17,23.43,33.67,23.65
 OpenCoder 8B,78.21,75.92,28.46,42.86,27.82,40.82,27.34,41.36,25.95,39.77,27.11,41.36,16.17,80.00,95.29,35.64,41.18,35.38,41.18,35.12,37.69,33.47,41.05,35.13,41.55
 SeedCoder 8B,91.41,85.31,53.46,47.35,53.33,46.53,52.86,49.42,50.62,45.60,51.65,49.59,28.23,77.44,94.12,37.31,30.59,37.31,27.06,37.32,23.53,35.35,26.92,36.89,27.23
-SeedCoder 8B Reasoning,67.82,53.47,49.23,30.20,49.23,29.39,48.92,32.04,46.76,28.64,47.87,29.99,-1,83.33,78.82,48.21,50.59,48.08,50.59,47.78,41.74,45.44,50.02,47.06,52.92
+SeedCoder 8B Reasoning,67.82,53.47,49.23,30.20,49.23,29.39,48.92,32.04,46.76,28.64,47.87,29.99,-1.00,83.33,78.82,48.21,50.59,48.08,50.59,47.78,41.74,45.44,50.02,47.06,52.92
 QwenCoder 2.5 7B,20.13,76.33,6.92,38.78,6.67,37.14,6.51,40.65,6.63,37.25,6.56,39.58,28.33,74.10,90.59,33.72,32.94,33.72,32.94,33.59,30.67,31.78,33.01,33.62,33.51
 "DeepSeek Coder 6,7B",82.05,78.78,29.62,41.22,29.49,38.78,29.51,42.62,27.73,39.33,29.41,43.30,24.63,67.18,84.71,31.67,24.71,29.87,24.71,29.78,23.53,27.98,24.50,29.21,24.79
 RTLCoder Mistral,54.87,32.24,24.62,16.33,24.62,15.92,24.28,16.03,22.78,14.71,24.06,16.00,14.77,60.51,85.88,27.05,36.47,27.05,36.47,26.94,34.63,25.22,36.55,26.87,37.64
 RTLCoder DeepSeek,84.62,73.06,39.49,37.14,39.49,34.69,38.91,34.30,37.52,32.76,38.55,33.69,19.35,77.31,85.88,36.92,40.00,36.79,40.00,36.94,35.57,34.84,39.83,36.62,39.60
 OriGen,96.15,81.63,54.23,50.61,54.23,50.61,54.29,53.10,51.57,50.86,53.15,53.44,17.07,92.44,98.82,50.77,58.82,50.77,58.82,50.95,54.14,48.53,58.81,50.51,61.40
-CodeV R1 Distill Qwen 7B,56.92,73.06,33.33,49.80,33.33,47.35,32.58,49.25,32.01,47.45,32.45,49.01,-1,92.69,89.41,21.28,49.41,21.28,49.41,21.04,43.68,19.59,49.06,21.05,49.91
+CodeV R1 Distill Qwen 7B,56.92,73.06,33.33,49.80,33.33,47.35,32.58,49.25,32.01,47.45,32.45,49.01,-1.00,92.69,89.41,21.28,49.41,21.28,49.41,21.04,43.68,19.59,49.06,21.05,49.91
 HaVen-CodeQwen,93.33,80.41,47.31,42.86,46.15,41.22,45.08,40.59,44.26,38.83,44.68,40.53,25.14,93.59,100.00,50.13,47.06,49.49,47.06,47.55,46.60,47.05,47.14,47.09,46.67
 CodeV-CL-7B,32.18,48.16,13.08,24.49,12.95,21.63,12.80,22.25,12.51,20.59,12.82,21.29,12.27,92.05,98.82,31.79,43.53,31.79,43.53,31.74,42.25,29.45,43.46,31.61,43.20
 CodeV-QW-7B,45.38,68.16,19.62,34.29,18.97,26.53,18.91,28.14,18.71,21.80,18.85,26.50,20.94,93.33,100.00,52.31,48.24,51.54,48.24,51.69,48.14,48.79,48.18,51.45,48.81

results/results.json

@@ -514,7 +514,7 @@
         "Model Type": "General",
         "Benchmark": "RTLLM",
         "Task": "Syntax (STX)",
-        "Result": 80.41,
+        "Result": 73.88,
         "Model URL": "https://huggingface.co/Qwen/Qwen3-235B-A22B",
         "Params": 235,
         "Release": "V2"
@@ -534,7 +534,7 @@
         "Model Type": "General",
         "Benchmark": "RTLLM",
         "Task": "Functionality (FNC)",
-        "Result": 53.06,
+        "Result": 51.43,
         "Model URL": "https://huggingface.co/Qwen/Qwen3-235B-A22B",
         "Params": 235,
         "Release": "V2"
@@ -554,7 +554,7 @@
         "Model Type": "General",
         "Benchmark": "RTLLM",
         "Task": "Synthesis (SYN)",
-        "Result": 51.43,
+        "Result": 48.57,
         "Model URL": "https://huggingface.co/Qwen/Qwen3-235B-A22B",
         "Params": 235,
         "Release": "V2"
@@ -574,7 +574,7 @@
         "Model Type": "General",
         "Benchmark": "RTLLM",
         "Task": "Power",
-        "Result": 57.77,
+        "Result": 54.61,
         "Model URL": "https://huggingface.co/Qwen/Qwen3-235B-A22B",
         "Params": 235,
         "Release": "V2"
@@ -594,7 +594,7 @@
         "Model Type": "General",
         "Benchmark": "RTLLM",
         "Task": "Performance",
-        "Result": 49.2,
+        "Result": 46.37,
         "Model URL": "https://huggingface.co/Qwen/Qwen3-235B-A22B",
         "Params": 235,
         "Release": "V2"
@@ -614,7 +614,7 @@
         "Model Type": "General",
         "Benchmark": "RTLLM",
         "Task": "Area",
-        "Result": 52.95,
+        "Result": 50.47,
         "Model URL": "https://huggingface.co/Qwen/Qwen3-235B-A22B",
         "Params": 235,
         "Release": "V2"
@@ -644,7 +644,7 @@
         "Model Type": "General",
         "Benchmark": "VeriGen",
         "Task": "Syntax (STX)",
-        "Result": 91.76,
+        "Result": 87.06,
         "Model URL": "https://huggingface.co/Qwen/Qwen3-235B-A22B",
         "Params": 235,
         "Release": "V2"
@@ -664,7 +664,7 @@
         "Model Type": "General",
         "Benchmark": "VeriGen",
         "Task": "Functionality (FNC)",
-        "Result": 40.0,
+        "Result": 49.41,
         "Model URL": "https://huggingface.co/Qwen/Qwen3-235B-A22B",
         "Params": 235,
         "Release": "V2"
@@ -684,7 +684,7 @@
         "Model Type": "General",
         "Benchmark": "VeriGen",
         "Task": "Synthesis (SYN)",
-        "Result": 40.0,
+        "Result": 49.41,
         "Model URL": "https://huggingface.co/Qwen/Qwen3-235B-A22B",
         "Params": 235,
         "Release": "V2"
@@ -704,7 +704,7 @@
         "Model Type": "General",
         "Benchmark": "VeriGen",
         "Task": "Power",
-        "Result": 39.09,
+        "Result": 41.82,
         "Model URL": "https://huggingface.co/Qwen/Qwen3-235B-A22B",
         "Params": 235,
         "Release": "V2"
@@ -724,7 +724,7 @@
         "Model Type": "General",
         "Benchmark": "VeriGen",
         "Task": "Performance",
-        "Result": 40.16,
+        "Result": 49.64,
         "Model URL": "https://huggingface.co/Qwen/Qwen3-235B-A22B",
         "Params": 235,
         "Release": "V2"
@@ -744,7 +744,7 @@
         "Model Type": "General",
         "Benchmark": "VeriGen",
         "Task": "Area",
-        "Result": 40.14,
+        "Result": 49.99,
         "Model URL": "https://huggingface.co/Qwen/Qwen3-235B-A22B",
         "Params": 235,
         "Release": "V2"