Update README.md

README.md

@@ -4,6 +4,488 @@ language:
 - ro
 base_model:
 - google/gemma-7b
+model-index:
+    - name: OpenLLM-Ro/RoGemma-7b-Instruct
+      results:
+        - task:
+            type: text-generation
+          dataset:
+            name: RoMT-Bench
+            type: RoMT-Bench
+          metrics:
+            - name: Score
+              type: Score
+              value: 5.26
+        - task:
+            type: text-generation
+          dataset:
+            name: RoCulturaBench
+            type: RoCulturaBench
+          metrics:
+            - name: Score
+              type: Score
+              value: 3.26
+        - task:
+            type: text-generation
+          dataset:
+            name: Romanian_Academic_Benchmarks
+            type: Romanian_Academic_Benchmarks
+          metrics:
+            - name: Average accuracy
+              type: accuracy
+              value: 53.41
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_arc_challenge
+            type: OpenLLM-Ro/ro_arc_challenge
+          metrics:
+            - name: Average accuracy
+              type: accuracy
+              value: 52.44
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_mmlu
+            type: OpenLLM-Ro/ro_mmlu
+          metrics:
+            - name: Average accuracy
+              type: accuracy
+              value: 54.44
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_winogrande
+            type: OpenLLM-Ro/ro_winogrande
+          metrics:
+            - name: Average accuracy
+              type: accuracy
+              value: 69.36
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_hellaswag
+            type: OpenLLM-Ro/ro_hellaswag
+          metrics:
+            - name: Average accuracy
+              type: accuracy
+              value: 61.96
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_gsm8k
+            type: OpenLLM-Ro/ro_gsm8k
+          metrics:
+            - name: Average accuracy
+              type: accuracy
+              value: 31.06
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_truthfulqa
+            type: OpenLLM-Ro/ro_truthfulqa
+          metrics:
+            - name: Average accuracy
+              type: accuracy
+              value: 51.23
+        - task:
+            type: text-generation
+          dataset:
+            name: LaRoSeDa_binary
+            type: LaRoSeDa_binary
+          metrics:
+            - name: Average macro-f1
+              type: macro-f1
+              value: 97.86
+        - task:
+            type: text-generation
+          dataset:
+            name: LaRoSeDa_multiclass
+            type: LaRoSeDa_multiclass
+          metrics:
+            - name: Average macro-f1
+              type: macro-f1
+              value: 65.70
+        - task:
+            type: text-generation
+          dataset:
+            name: LaRoSeDa_binary_finetuned
+            type: LaRoSeDa_binary_finetuned
+          metrics:
+            - name: Average macro-f1
+              type: macro-f1
+              value: 98.43
+        - task:
+            type: text-generation
+          dataset:
+            name: LaRoSeDa_multiclass_finetuned
+            type: LaRoSeDa_multiclass_finetuned
+          metrics:
+            - name: Average macro-f1
+              type: macro-f1
+              value: 87.17
+        - task:
+            type: text-generation
+          dataset:
+            name: WMT_EN-RO
+            type: WMT_EN-RO
+          metrics:
+            - name: Average bleu
+              type: bleu
+              value: 27.91
+        - task:
+            type: text-generation
+          dataset:
+            name: WMT_RO-EN
+            type: WMT_RO-EN
+          metrics:
+            - name: Average bleu
+              type: bleu
+              value: 23.08
+        - task:
+            type: text-generation
+          dataset:
+            name: WMT_EN-RO_finetuned
+            type: WMT_EN-RO_finetuned
+          metrics:
+            - name: Average bleu
+              type: bleu
+              value: 27.99
+        - task:
+            type: text-generation
+          dataset:
+            name: WMT_RO-EN_finetuned
+            type: WMT_RO-EN_finetuned
+          metrics:
+            - name: Average bleu
+              type: bleu
+              value: 39.51
+        - task:
+            type: text-generation
+          dataset:
+            name: XQuAD
+            type: XQuAD
+          metrics:
+            - name: Average exact_match
+              type: exact_match
+              value: 17.75
+        - task:
+            type: text-generation
+          dataset:
+            name: XQuAD
+            type: XQuAD
+          metrics:
+            - name: Average f1
+              type: f1
+              value: 28.11
+        - task:
+            type: text-generation
+          dataset:
+            name: XQuAD_finetuned
+            type: XQuAD_finetuned
+          metrics:
+            - name: Average exact_match
+              type: exact_match
+              value: 52.02
+        - task:
+            type: text-generation
+          dataset:
+            name: XQuAD_finetuned
+            type: XQuAD_finetuned
+          metrics:
+            - name: Average f1
+              type: f1
+              value: 68.43
+        - task:
+            type: text-generation
+          dataset:
+            name: STS
+            type: STS
+          metrics:
+            - name: Average spearman
+              type: spearman
+              value: 73.96
+        - task:
+            type: text-generation
+          dataset:
+            name: STS
+            type: STS
+          metrics:
+            - name: Average pearson
+              type: pearson
+              value: 75.16
+        - task:
+            type: text-generation
+          dataset:
+            name: STS_finetuned
+            type: STS_finetuned
+          metrics:
+            - name: Average spearman
+              type: spearman
+              value: 86.45
+        - task:
+            type: text-generation
+          dataset:
+            name: STS_finetuned
+            type: STS_finetuned
+          metrics:
+            - name: Average pearson
+              type: pearson
+              value: 86.31
+        - task:
+            type: text-generation
+          dataset:
+            name: RoMT-Bench
+            type: RoMT-Bench
+          metrics:
+            - name: First turn
+              type: Score
+              value: 5.92
+            - name: Second turn
+              type: Score
+              value: 4.60
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_arc_challenge
+            type: OpenLLM-Ro/ro_arc_challenge
+          metrics:
+            - name: 0-shot 
+              type: accuracy
+              value: 50.30
+            - name: 1-shot 
+              type: accuracy
+              value: 50.90
+            - name: 3-shot 
+              type: accuracy
+              value: 52.53
+            - name: 5-shot 
+              type: accuracy
+              value: 53.30
+            - name: 10-shot 
+              type: accuracy
+              value: 54.33
+            - name: 25-shot 
+              type: accuracy
+              value: 53.30
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_mmlu
+            type: OpenLLM-Ro/ro_mmlu
+          metrics:
+            - name: 0-shot 
+              type: accuracy
+              value: 54.95
+            - name: 1-shot 
+              type: accuracy
+              value: 54.01
+            - name: 3-shot 
+              type: accuracy
+              value: 54.03
+            - name: 5-shot 
+              type: accuracy
+              value: 54.76
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_winogrande
+            type: OpenLLM-Ro/ro_winogrande
+          metrics:
+            - name: 0-shot 
+              type: accuracy
+              value: 68.67
+            - name: 1-shot 
+              type: accuracy
+              value: 69.46
+            - name: 3-shot 
+              type: accuracy
+              value: 68.43
+            - name: 5-shot 
+              type: accuracy
+              value: 70.88
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_hellaswag
+            type: OpenLLM-Ro/ro_hellaswag
+          metrics:
+            - name: 0-shot 
+              type: accuracy
+              value: 61.54
+            - name: 1-shot 
+              type: accuracy
+              value: 61.54
+            - name: 3-shot 
+              type: accuracy
+              value: 62.08
+            - name: 5-shot 
+              type: accuracy
+              value: 62.12
+            - name: 10-shot 
+              type: accuracy
+              value: 62.51
+        - task:
+            type: text-generation
+          dataset:
+            name: OpenLLM-Ro/ro_gsm8k
+            type: OpenLLM-Ro/ro_gsm8k
+          metrics:
+            - name: 0-shot 
+              type: accuracy
+              value: 24.79
+            - name: 1-shot 
+              type: accuracy
+              value: 34.50
+            - name: 3-shot 
+              type: accuracy
+              value: 33.89
+        - task:
+            type: text-generation
+          dataset:
+            name: LaRoSeDa_binary
+            type: LaRoSeDa_binary
+          metrics:
+            - name: 0-shot 
+              type: macro-f1
+              value: 97.60
+            - name: 1-shot 
+              type: macro-f1
+              value: 97.23
+            - name: 3-shot 
+              type: macro-f1
+              value: 98.13
+            - name: 5-shot 
+              type: macro-f1
+              value: 98.50
+        - task:
+            type: text-generation
+          dataset:
+            name: LaRoSeDa_multiclass
+            type: LaRoSeDa_multiclass
+          metrics:
+            - name: 0-shot 
+              type: macro-f1
+              value: 68.53
+            - name: 1-shot 
+              type: macro-f1
+              value: 64.84
+            - name: 3-shot 
+              type: macro-f1
+              value: 63.62
+            - name: 5-shot 
+              type: macro-f1
+              value: 65.83
+        - task:
+            type: text-generation
+          dataset:
+            name: WMT_EN-RO
+            type: WMT_EN-RO
+          metrics:
+            - name: 0-shot 
+              type: bleu
+              value: 25.04
+            - name: 1-shot 
+              type: bleu
+              value: 28.43
+            - name: 3-shot 
+              type: bleu
+              value: 28.87
+            - name: 5-shot 
+              type: bleu
+              value: 29.28
+        - task:
+            type: text-generation
+          dataset:
+            name: WMT_RO-EN
+            type: WMT_RO-EN
+          metrics:
+            - name: 0-shot 
+              type: bleu
+              value: 4.94
+            - name: 1-shot 
+              type: bleu
+              value: 25.33
+            - name: 3-shot 
+              type: bleu
+              value: 30.87
+            - name: 5-shot 
+              type: bleu
+              value: 31.19
+        - task:
+            type: text-generation
+          dataset:
+            name: XQuAD_EM
+            type: XQuAD_EM
+          metrics:
+            - name: 0-shot 
+              type: exact_match
+              value: 36.47
+            - name: 1-shot 
+              type: exact_match
+              value: 26.22
+            - name: 3-shot 
+              type: exact_match
+              value: 3.19
+            - name: 5-shot 
+              type: exact_match
+              value: 5.13
+        - task:
+            type: text-generation
+          dataset:
+            name: XQuAD_F1
+            type: XQuAD_F1
+          metrics:
+            - name: 0-shot 
+              type: f1
+              value: 56.83
+            - name: 1-shot 
+              type: f1
+              value: 38.53
+            - name: 3-shot 
+              type: f1
+              value: 6.88
+            - name: 5-shot 
+              type: f1
+              value: 10.19
+        - task:
+            type: text-generation
+          dataset:
+            name: STS
+            type: STS
+          metrics:
+            - name: 0-shot 
+              type: spearman
+              value: 70.61
+            - name: 1-shot 
+              type: spearman
+              value: 73.53
+            - name: 3-shot 
+              type: spearman
+              value: 77.73
+        - task:
+            type: text-generation
+          dataset:
+            name: STS
+            type: STS
+          metrics:
+            - name: 0-shot 
+              type: pearson
+              value: 72.28
+            - name: 1-shot 
+              type: pearson
+              value: 74.46
+            - name: 3-shot 
+              type: pearson
+              value: 78.75
+datasets:
+- OpenLLM-Ro/ro_sft_alpaca
+- OpenLLM-Ro/ro_sft_alpaca_gpt4
+- OpenLLM-Ro/ro_sft_dolly
+- OpenLLM-Ro/ro_sft_selfinstruct_gpt4
+- OpenLLM-Ro/ro_sft_norobots
+- OpenLLM-Ro/ro_sft_orca
+- OpenLLM-Ro/ro_sft_camel
 ---
 
 # Model Card for Model ID
@@ -90,7 +572,7 @@ print(tokenizer.decode(outputs[0]))
 <td>gemma-1.1-7b-it</td><td><center>41.44</center></td><td><center>40.32</center></td><td><center>47.22</center></td><td><center>55.01</center></td><td><center>47.03</center></td><td><center>9.50</center></td><td><center>49.58</center></td>
 </tr>
 <tr>
-<td><em>RoGemma-7b-Instruct</em></td><td><center><em><strong>53.42</strong></em></center></td><td><center><em><strong>52.44</strong></em></center></td><td><center><em><strong>54.44</strong></em></center></td><td><center><em><strong>69.36</strong></em></center></td><td><center><em><strong>61.96</strong></em></center></td><td><center><em><strong>31.06</strong></em></center></td><td><center><em><strong>51.23</strong></em></center></td>
+<td><em>RoGemma-7b-Instruct</em></td><td><center><em><strong>53.41</strong></em></center></td><td><center><em><strong>52.44</strong></em></center></td><td><center><em><strong>54.44</strong></em></center></td><td><center><em><strong>69.36</strong></em></center></td><td><center><em><strong>61.96</strong></em></center></td><td><center><em><strong>31.06</strong></em></center></td><td><center><em><strong>51.23</strong></em></center></td>
 </tr>
 </tbody>
 </table>
@@ -123,15 +605,16 @@ print(tokenizer.decode(outputs[0]))
 <td><center><strong>RO-EN<br>(Bleu)</strong></center>
 </tr>
 <tr>
-<td>gemma-1.1-7b-it</td><td><center>87.54</center></td><td><center>51.49</center></td><td><center>83.87</center></td><td><center>85.61</center></td><td><center>17.96</center></td><td><center><strong>27.74</strong></center></td><td><center>25.48</center></td><td><center>36.11</center></td>
+<td>gemma-1.1-7b-it</td><td><center>87.54</center></td><td><center>51.48</center></td><td><center>83.87</center></td><td><center>85.61</center></td><td><center>17.96</center></td><td><center><strong>27.74</strong></center></td><td><center>25.48</center></td><td><center>36.11</center></td>
 </tr>
 <tr>
-<td><em>RoGemma-7b-Instruct</em></td><td><center><em><strong>97.87</strong></em></center></td><td><center><em><strong>65.71</strong></em></center></td><td><center><em><strong>98.43</strong></em></center></td><td><center><em><strong>87.18</strong></em></center></td><td><center><em><strong>27.91</strong></em></center></td><td><center><em>23.08</em></center></td><td><center><em><strong>27.99</strong></em></center></td><td><center><em><strong>39.51</strong></em></center></td>
+<td><em>RoGemma-7b-Instruct</em></td><td><center><em><strong>97.86</strong></em></center></td><td><center><em><strong>65.70</strong></em></center></td><td><center><em><strong>98.43</strong></em></center></td><td><center><em><strong>87.17</strong></em></center></td><td><center><em><strong>27.91</strong></em></center></td><td><center><em>23.08</em></center></td><td><center><em><strong>27.99</strong></em></center></td><td><center><em><strong>39.51</strong></em></center></td>
 </tr>
 </tbody>
 </table>
 
 
+
 <table>
 <tbody>
 <tr>
@@ -158,7 +641,7 @@ print(tokenizer.decode(outputs[0]))
 <td><center><strong>(Pearson)</strong></center></td>
 </tr>
 <tr>
-<td>gemma-1.1-7b-it</td><td><center><strong>42.10</strong></center></td><td><center><strong>62.30</strong></center></td><td><center><strong>60.34</strong></center></td><td><center><strong>77.40</strong></center></td><td><center>49.10</center></td><td><center>50.23</center></td><td><center>83.43</center></td><td><center>83.65</center></td>
+<td>gemma-1.1-7b-it</td><td><center><strong>42.10</strong></center></td><td><center><strong>62.30</strong></center></td><td><center><strong>60.34</strong></center></td><td><center><strong>77.40</strong></center></td><td><center>49.10</center></td><td><center>50.23</center></td><td><center>83.43</center></td><td><center>83.64</center></td>
 </tr>
 <tr>
 <td><em>RoGemma-7b-Instruct</em></td><td><center><em>17.75</em></center></td><td><center><em>28.11</em></center></td><td><center><em>52.02</em></center></td><td><center><em>68.43</em></center></td><td><center><em><strong>73.96</strong></em></center></td><td><center><em><strong>75.16</strong></em></center></td><td><center><em><strong>86.45</strong></em></center></td><td><center><em><strong>86.31</strong></em></center></td>