diff --git "a/README.md" "b/README.md" --- "a/README.md" +++ "b/README.md" @@ -1,10046 +1,140 @@ --- -model-index: -- name: giga-embeddings-instruct - results: - - dataset: - config: en-ext - name: MTEB AmazonCounterfactualClassification (en-ext) - revision: e8379541af4e31359cca9fbcf4b00f2671dba205 - split: test - type: mteb/amazon_counterfactual - metrics: - - type: accuracy - value: 94.5352323838081 - - type: ap - value: 62.422648408367344 - - type: ap_weighted - value: 62.422648408367344 - - type: f1 - value: 87.13103677336655 - - type: f1_weighted - value: 94.85637995412655 - - type: main_score - value: 94.5352323838081 - task: - type: Classification - - dataset: - config: en - name: MTEB AmazonCounterfactualClassification (en) - revision: e8379541af4e31359cca9fbcf4b00f2671dba205 - split: test - type: mteb/amazon_counterfactual - metrics: - - type: accuracy - value: 90.31343283582089 - - type: ap - value: 63.42364739316405 - - type: ap_weighted - value: 63.42364739316405 - - type: f1 - value: 85.54214552412623 - - type: f1_weighted - value: 90.59539168268289 - - type: main_score - value: 90.31343283582089 - task: - type: Classification - - dataset: - config: default - name: MTEB AmazonPolarityClassification (default) - revision: e2d317d38cd51312af73b3d32a06d1a08b442046 - split: test - type: mteb/amazon_polarity - metrics: - - type: accuracy - value: 94.29605000000001 - - type: ap - value: 91.30887530384256 - - type: ap_weighted - value: 91.30887530384256 - - type: f1 - value: 94.29070662237378 - - type: f1_weighted - value: 94.29070662237378 - - type: main_score - value: 94.29605000000001 - task: - type: Classification - - dataset: - config: default - name: MTEB ArguAna (default) - revision: c22ab2a51041ffd869aaddef7af8d8215647e41a - split: test - type: mteb/arguana - metrics: - - type: main_score - value: 53.227999999999994 - - type: map_at_1 - value: 27.595999999999997 - - type: map_at_10 - value: 43.756 - - type: map_at_100 - value: 44.674 - - type: map_at_1000 - value: 44.675 - - type: map_at_20 - value: 44.511 - - type: map_at_3 - value: 38.312000000000005 - - type: map_at_5 - value: 41.271 - - type: mrr_at_1 - value: 27.951635846372692 - - type: mrr_at_10 - value: 43.8683138025244 - - type: mrr_at_100 - value: 44.79916793634115 - - type: mrr_at_1000 - value: 44.800641832434614 - - type: mrr_at_20 - value: 44.63636850959653 - - type: mrr_at_3 - value: 38.383119962067305 - - type: mrr_at_5 - value: 41.41299193930774 - - type: nauc_map_at_1000_diff1 - value: 6.936710279308449 - - type: nauc_map_at_1000_max - value: -16.426102328143827 - - type: nauc_map_at_1000_std - value: -18.408713623781154 - - type: nauc_map_at_100_diff1 - value: 6.936900325690782 - - type: nauc_map_at_100_max - value: -16.424599448813982 - - type: nauc_map_at_100_std - value: -18.41002427519262 - - type: nauc_map_at_10_diff1 - value: 6.686089466049945 - - type: nauc_map_at_10_max - value: -16.277854443721235 - - type: nauc_map_at_10_std - value: -18.533367246025183 - - type: nauc_map_at_1_diff1 - value: 10.048892770421086 - - type: nauc_map_at_1_max - value: -18.88033774058785 - - type: nauc_map_at_1_std - value: -18.950654138263662 - - type: nauc_map_at_20_diff1 - value: 6.896257398324564 - - type: nauc_map_at_20_max - value: -16.28720522758851 - - type: nauc_map_at_20_std - value: -18.463554340157874 - - type: nauc_map_at_3_diff1 - value: 6.996349008138944 - - type: nauc_map_at_3_max - value: -16.895326699141894 - - type: nauc_map_at_3_std - value: -18.550696483491105 - - type: nauc_map_at_5_diff1 - value: 6.652257808997529 - - type: nauc_map_at_5_max - value: -16.616340120756664 - - type: nauc_map_at_5_std - value: -18.750380766744815 - - type: nauc_mrr_at_1000_diff1 - value: 5.675242976111991 - - type: nauc_mrr_at_1000_max - value: -16.992812047837067 - - type: nauc_mrr_at_1000_std - value: -18.32929497132872 - - type: nauc_mrr_at_100_diff1 - value: 5.6754937777142835 - - type: nauc_mrr_at_100_max - value: -16.991287123334946 - - type: nauc_mrr_at_100_std - value: -18.330604638796043 - - type: nauc_mrr_at_10_diff1 - value: 5.392768177635316 - - type: nauc_mrr_at_10_max - value: -16.891663162548255 - - type: nauc_mrr_at_10_std - value: -18.471864534496945 - - type: nauc_mrr_at_1_diff1 - value: 8.923777873913467 - - type: nauc_mrr_at_1_max - value: -18.81665268664494 - - type: nauc_mrr_at_1_std - value: -18.819665466571674 - - type: nauc_mrr_at_20_diff1 - value: 5.641752338928701 - - type: nauc_mrr_at_20_max - value: -16.85136568990159 - - type: nauc_mrr_at_20_std - value: -18.384362648232546 - - type: nauc_mrr_at_3_diff1 - value: 5.524316132813568 - - type: nauc_mrr_at_3_max - value: -17.723568343459988 - - type: nauc_mrr_at_3_std - value: -18.372688451025656 - - type: nauc_mrr_at_5_diff1 - value: 5.414405183203325 - - type: nauc_mrr_at_5_max - value: -17.288127460794154 - - type: nauc_mrr_at_5_std - value: -18.71123050851349 - - type: nauc_ndcg_at_1000_diff1 - value: 6.487802962417493 - - type: nauc_ndcg_at_1000_max - value: -15.76159401306176 - - type: nauc_ndcg_at_1000_std - value: -18.15838595665605 - - type: nauc_ndcg_at_100_diff1 - value: 6.48323468898899 - - type: nauc_ndcg_at_100_max - value: -15.728467477722477 - - type: nauc_ndcg_at_100_std - value: -18.197384218078643 - - type: nauc_ndcg_at_10_diff1 - value: 5.423448018411026 - - type: nauc_ndcg_at_10_max - value: -14.673502378215453 - - type: nauc_ndcg_at_10_std - value: -18.837931889895316 - - type: nauc_ndcg_at_1_diff1 - value: 10.048892770421086 - - type: nauc_ndcg_at_1_max - value: -18.88033774058785 - - type: nauc_ndcg_at_1_std - value: -18.950654138263662 - - type: nauc_ndcg_at_20_diff1 - value: 6.369954849420038 - - type: nauc_ndcg_at_20_max - value: -14.443991776264713 - - type: nauc_ndcg_at_20_std - value: -18.416264332865836 - - type: nauc_ndcg_at_3_diff1 - value: 6.224331563078568 - - type: nauc_ndcg_at_3_max - value: -16.183370694913553 - - type: nauc_ndcg_at_3_std - value: -18.559481650690337 - - type: nauc_ndcg_at_5_diff1 - value: 5.659342042143408 - - type: nauc_ndcg_at_5_max - value: -15.510631438356693 - - type: nauc_ndcg_at_5_std - value: -18.909647623269873 - - type: nauc_precision_at_1000_diff1 - value: -45.740924328524436 - - type: nauc_precision_at_1000_max - value: -4.436745319184523 - - type: nauc_precision_at_1000_std - value: 57.94428979357973 - - type: nauc_precision_at_100_diff1 - value: -23.751971897164438 - - type: nauc_precision_at_100_max - value: 0.5109176204949021 - - type: nauc_precision_at_100_std - value: 14.133130213074722 - - type: nauc_precision_at_10_diff1 - value: -2.2741922400170953 - - type: nauc_precision_at_10_max - value: -4.695134136659869 - - type: nauc_precision_at_10_std - value: -21.566024184206757 - - type: nauc_precision_at_1_diff1 - value: 10.048892770421086 - - type: nauc_precision_at_1_max - value: -18.88033774058785 - - type: nauc_precision_at_1_std - value: -18.950654138263662 - - type: nauc_precision_at_20_diff1 - value: 2.366832261816588 - - type: nauc_precision_at_20_max - value: 17.078759245976265 - - type: nauc_precision_at_20_std - value: -17.573684824976628 - - type: nauc_precision_at_3_diff1 - value: 4.062538060385958 - - type: nauc_precision_at_3_max - value: -14.10949953336873 - - type: nauc_precision_at_3_std - value: -18.626114079282416 - - type: nauc_precision_at_5_diff1 - value: 2.425834990396102 - - type: nauc_precision_at_5_max - value: -11.600278541101094 - - type: nauc_precision_at_5_std - value: -19.53326796179894 - - type: nauc_recall_at_1000_diff1 - value: -45.740924328527974 - - type: nauc_recall_at_1000_max - value: -4.4367453191877555 - - type: nauc_recall_at_1000_std - value: 57.9442897935769 - - type: nauc_recall_at_100_diff1 - value: -23.751971897160466 - - type: nauc_recall_at_100_max - value: 0.5109176204928446 - - type: nauc_recall_at_100_std - value: 14.133130213071956 - - type: nauc_recall_at_10_diff1 - value: -2.2741922400170527 - - type: nauc_recall_at_10_max - value: -4.695134136659742 - - type: nauc_recall_at_10_std - value: -21.566024184206647 - - type: nauc_recall_at_1_diff1 - value: 10.048892770421086 - - type: nauc_recall_at_1_max - value: -18.88033774058785 - - type: nauc_recall_at_1_std - value: -18.950654138263662 - - type: nauc_recall_at_20_diff1 - value: 2.366832261816872 - - type: nauc_recall_at_20_max - value: 17.078759245976432 - - type: nauc_recall_at_20_std - value: -17.57368482497646 - - type: nauc_recall_at_3_diff1 - value: 4.0625380603860055 - - type: nauc_recall_at_3_max - value: -14.10949953336872 - - type: nauc_recall_at_3_std - value: -18.626114079282395 - - type: nauc_recall_at_5_diff1 - value: 2.425834990396135 - - type: nauc_recall_at_5_max - value: -11.60027854110106 - - type: nauc_recall_at_5_std - value: -19.533267961798924 - - type: ndcg_at_1 - value: 27.595999999999997 - - type: ndcg_at_10 - value: 53.227999999999994 - - type: ndcg_at_100 - value: 56.931 - - type: ndcg_at_1000 - value: 56.967999999999996 - - type: ndcg_at_20 - value: 55.921 - - type: ndcg_at_3 - value: 41.908 - - type: ndcg_at_5 - value: 47.285 - - type: precision_at_1 - value: 27.595999999999997 - - type: precision_at_10 - value: 8.371 - - type: precision_at_100 - value: 0.9939999999999999 - - type: precision_at_1000 - value: 0.1 - - type: precision_at_20 - value: 4.712000000000001 - - type: precision_at_3 - value: 17.449 - - type: precision_at_5 - value: 13.100999999999999 - - type: recall_at_1 - value: 27.595999999999997 - - type: recall_at_10 - value: 83.71300000000001 - - type: recall_at_100 - value: 99.36 - - type: recall_at_1000 - value: 99.644 - - type: recall_at_20 - value: 94.23899999999999 - - type: recall_at_3 - value: 52.347 - - type: recall_at_5 - value: 65.505 - task: - type: Retrieval - - dataset: - config: default - name: MTEB AskUbuntuDupQuestions (default) - revision: 2000358ca161889fa9c082cb41daa8dcfb161a54 - split: test - type: mteb/askubuntudupquestions-reranking - metrics: - - type: main_score - value: 58.18725222465893 - - type: map - value: 58.18725222465893 - - type: mrr - value: 69.84335839598998 - - type: nAUC_map_diff1 - value: 19.11598989756231 - - type: nAUC_map_max - value: 15.695053858587466 - - type: nAUC_map_std - value: 22.147773436080342 - - type: nAUC_mrr_diff1 - value: 25.38427130339339 - - type: nAUC_mrr_max - value: 24.2962940173052 - - type: nAUC_mrr_std - value: 20.846599304343176 - task: - type: Reranking - - dataset: - config: default - name: MTEB BIOSSES (default) - revision: d3fb88f8f02e40887cd149695127462bbcf29b4a - split: test - type: mteb/biosses-sts - metrics: - - type: cosine_pearson - value: 87.40814966131607 - - type: cosine_spearman - value: 84.21758160533057 - - type: euclidean_pearson - value: 86.68087011664755 - - type: euclidean_spearman - value: 84.21758160533057 - - type: main_score - value: 84.21758160533057 - - type: manhattan_pearson - value: 86.8885717540405 - - type: manhattan_spearman - value: 84.69409848718736 - - type: pearson - value: 87.40814966131607 - - type: spearman - value: 84.21758160533057 - task: - type: STS - - dataset: - config: default - name: MTEB Banking77Classification (default) - revision: 0fd18e25b25c072e09e0d92ab615fda904d66300 - split: test - type: mteb/banking77 - metrics: - - type: accuracy - value: 87.78571428571429 - - type: f1 - value: 87.55183393575304 - - type: f1_weighted - value: 87.55183393575307 - - type: main_score - value: 87.78571428571429 - task: - type: Classification - - dataset: - config: default - name: MTEB CEDRClassification (default) - revision: c0ba03d058e3e1b2f3fd20518875a4563dd12db4 - split: test - type: ai-forever/cedr-classification - metrics: - - type: accuracy - value: 59.6894792773645 - - type: f1 - value: 59.07371458842751 - - type: lrap - value: 84.46838469713137 - - type: main_score - value: 59.6894792773645 - task: - type: MultilabelClassification - - dataset: - config: default - name: MTEB CQADupstackAndroidRetrieval (default) - revision: f46a197baaae43b4f621051089b82a364682dfeb - split: test - type: mteb/cqadupstack-android - metrics: - - type: main_score - value: 36.264 - - type: map_at_1 - value: 21.54 - - type: map_at_10 - value: 30.146 - - type: map_at_100 - value: 31.733 - - type: map_at_1000 - value: 31.909 - - type: map_at_20 - value: 30.973 - - type: map_at_3 - value: 27.021 - - type: map_at_5 - value: 28.444000000000003 - - type: mrr_at_1 - value: 27.181688125894134 - - type: mrr_at_10 - value: 36.026636691872724 - - type: mrr_at_100 - value: 37.03343694715993 - - type: mrr_at_1000 - value: 37.089502279951105 - - type: mrr_at_20 - value: 36.575238947911984 - - type: mrr_at_3 - value: 33.07105388650455 - - type: mrr_at_5 - value: 34.50882212684788 - - type: nauc_map_at_1000_diff1 - value: 40.00591947815747 - - type: nauc_map_at_1000_max - value: 24.149847825797917 - - type: nauc_map_at_1000_std - value: 3.2086670255845404 - - type: nauc_map_at_100_diff1 - value: 40.00441108717158 - - type: nauc_map_at_100_max - value: 24.055155307330164 - - type: nauc_map_at_100_std - value: 3.2204001395328916 - - type: nauc_map_at_10_diff1 - value: 39.92034196847358 - - type: nauc_map_at_10_max - value: 23.481367728480564 - - type: nauc_map_at_10_std - value: 2.46524430460527 - - type: nauc_map_at_1_diff1 - value: 45.210951522814256 - - type: nauc_map_at_1_max - value: 23.426091924328706 - - type: nauc_map_at_1_std - value: -2.7839519807285074 - - type: nauc_map_at_20_diff1 - value: 39.765141613442495 - - type: nauc_map_at_20_max - value: 23.8640869346293 - - type: nauc_map_at_20_std - value: 2.9991581054265746 - - type: nauc_map_at_3_diff1 - value: 40.6459399697723 - - type: nauc_map_at_3_max - value: 23.680185542835098 - - type: nauc_map_at_3_std - value: 0.29419617901594664 - - type: nauc_map_at_5_diff1 - value: 39.93079810788306 - - type: nauc_map_at_5_max - value: 23.524497440705254 - - type: nauc_map_at_5_std - value: 1.7962358900605613 - - type: nauc_mrr_at_1000_diff1 - value: 39.10944438111878 - - type: nauc_mrr_at_1000_max - value: 24.778123336724732 - - type: nauc_mrr_at_1000_std - value: 3.0165840634640846 - - type: nauc_mrr_at_100_diff1 - value: 39.10734982136307 - - type: nauc_mrr_at_100_max - value: 24.766616675099815 - - type: nauc_mrr_at_100_std - value: 3.0334155163562615 - - type: nauc_mrr_at_10_diff1 - value: 39.037151411366814 - - type: nauc_mrr_at_10_max - value: 24.599134492259704 - - type: nauc_mrr_at_10_std - value: 2.978930527526268 - - type: nauc_mrr_at_1_diff1 - value: 44.96218824994712 - - type: nauc_mrr_at_1_max - value: 25.591689538899846 - - type: nauc_mrr_at_1_std - value: -1.3310616968087603 - - type: nauc_mrr_at_20_diff1 - value: 38.909044438924994 - - type: nauc_mrr_at_20_max - value: 24.621874344604244 - - type: nauc_mrr_at_20_std - value: 2.965642154907867 - - type: nauc_mrr_at_3_diff1 - value: 39.36364809687436 - - type: nauc_mrr_at_3_max - value: 24.843395616218665 - - type: nauc_mrr_at_3_std - value: 1.3819897830014731 - - type: nauc_mrr_at_5_diff1 - value: 38.863436520404 - - type: nauc_mrr_at_5_max - value: 24.90556151740949 - - type: nauc_mrr_at_5_std - value: 2.7820696970598653 - - type: nauc_ndcg_at_1000_diff1 - value: 38.83322428704425 - - type: nauc_ndcg_at_1000_max - value: 25.27156294182745 - - type: nauc_ndcg_at_1000_std - value: 6.973933838132665 - - type: nauc_ndcg_at_100_diff1 - value: 38.40151337957579 - - type: nauc_ndcg_at_100_max - value: 24.160017604166402 - - type: nauc_ndcg_at_100_std - value: 7.235920148334983 - - type: nauc_ndcg_at_10_diff1 - value: 37.50309281936751 - - type: nauc_ndcg_at_10_max - value: 23.024367502884033 - - type: nauc_ndcg_at_10_std - value: 4.995596569024126 - - type: nauc_ndcg_at_1_diff1 - value: 44.96218824994712 - - type: nauc_ndcg_at_1_max - value: 25.591689538899846 - - type: nauc_ndcg_at_1_std - value: -1.3310616968087603 - - type: nauc_ndcg_at_20_diff1 - value: 36.84316736441035 - - type: nauc_ndcg_at_20_max - value: 23.35930351746899 - - type: nauc_ndcg_at_20_std - value: 5.764156579665355 - - type: nauc_ndcg_at_3_diff1 - value: 39.00520993808036 - - type: nauc_ndcg_at_3_max - value: 24.336089017831654 - - type: nauc_ndcg_at_3_std - value: 1.9558269186682105 - - type: nauc_ndcg_at_5_diff1 - value: 37.76541751097454 - - type: nauc_ndcg_at_5_max - value: 23.93546553994223 - - type: nauc_ndcg_at_5_std - value: 4.178781488281662 - - type: nauc_precision_at_1000_diff1 - value: -10.372529546949247 - - type: nauc_precision_at_1000_max - value: 3.594993364396436 - - type: nauc_precision_at_1000_std - value: -3.360438126279742 - - type: nauc_precision_at_100_diff1 - value: 3.5552491090605667 - - type: nauc_precision_at_100_max - value: 13.448701405797095 - - type: nauc_precision_at_100_std - value: 12.621615292982941 - - type: nauc_precision_at_10_diff1 - value: 19.337662094452096 - - type: nauc_precision_at_10_max - value: 22.307700535828424 - - type: nauc_precision_at_10_std - value: 14.350516411235187 - - type: nauc_precision_at_1_diff1 - value: 44.96218824994712 - - type: nauc_precision_at_1_max - value: 25.591689538899846 - - type: nauc_precision_at_1_std - value: -1.3310616968087603 - - type: nauc_precision_at_20_diff1 - value: 12.196602490339437 - - type: nauc_precision_at_20_max - value: 21.283209534212133 - - type: nauc_precision_at_20_std - value: 15.740621879710156 - - type: nauc_precision_at_3_diff1 - value: 31.369103118497083 - - type: nauc_precision_at_3_max - value: 25.298131418390064 - - type: nauc_precision_at_3_std - value: 6.505279304669079 - - type: nauc_precision_at_5_diff1 - value: 26.088097622906997 - - type: nauc_precision_at_5_max - value: 25.166926352778273 - - type: nauc_precision_at_5_std - value: 12.204299883327282 - - type: nauc_recall_at_1000_diff1 - value: 34.255882215302044 - - type: nauc_recall_at_1000_max - value: 27.835351170558503 - - type: nauc_recall_at_1000_std - value: 59.56089209650579 - - type: nauc_recall_at_100_diff1 - value: 28.507339884100354 - - type: nauc_recall_at_100_max - value: 16.4950314804097 - - type: nauc_recall_at_100_std - value: 23.991797968783846 - - type: nauc_recall_at_10_diff1 - value: 29.44195661453061 - - type: nauc_recall_at_10_max - value: 16.630540783772904 - - type: nauc_recall_at_10_std - value: 10.542317179779532 - - type: nauc_recall_at_1_diff1 - value: 45.210951522814256 - - type: nauc_recall_at_1_max - value: 23.426091924328706 - - type: nauc_recall_at_1_std - value: -2.7839519807285074 - - type: nauc_recall_at_20_diff1 - value: 25.291470975834052 - - type: nauc_recall_at_20_max - value: 16.973739781631583 - - type: nauc_recall_at_20_std - value: 12.628456973580771 - - type: nauc_recall_at_3_diff1 - value: 34.530232948887644 - - type: nauc_recall_at_3_max - value: 21.410528878364 - - type: nauc_recall_at_3_std - value: 3.3335643912576365 - - type: nauc_recall_at_5_diff1 - value: 31.549298686623896 - - type: nauc_recall_at_5_max - value: 20.019545741301524 - - type: nauc_recall_at_5_std - value: 8.638769848309126 - - type: ndcg_at_1 - value: 27.182000000000002 - - type: ndcg_at_10 - value: 36.264 - - type: ndcg_at_100 - value: 42.935 - - type: ndcg_at_1000 - value: 45.64 - - type: ndcg_at_20 - value: 38.619 - - type: ndcg_at_3 - value: 30.971 - - type: ndcg_at_5 - value: 32.799 - - type: precision_at_1 - value: 27.182000000000002 - - type: precision_at_10 - value: 7.396 - - type: precision_at_100 - value: 1.375 - - type: precision_at_1000 - value: 0.20600000000000002 - - type: precision_at_20 - value: 4.571 - - type: precision_at_3 - value: 15.068999999999999 - - type: precision_at_5 - value: 10.959000000000001 - - type: recall_at_1 - value: 21.54 - - type: recall_at_10 - value: 48.304 - - type: recall_at_100 - value: 77.24 - - type: recall_at_1000 - value: 94.405 - - type: recall_at_20 - value: 56.845 - - type: recall_at_3 - value: 32.828 - - type: recall_at_5 - value: 38.029 - task: - type: Retrieval - - dataset: - config: default - name: MTEB CQADupstackEnglishRetrieval (default) - revision: ad9991cb51e31e31e430383c75ffb2885547b5f0 - split: test - type: mteb/cqadupstack-english - metrics: - - type: main_score - value: 32.666000000000004 - - type: map_at_1 - value: 19.496 - - type: map_at_10 - value: 27.708 - - type: map_at_100 - value: 28.884999999999998 - - type: map_at_1000 - value: 29.03 - - type: map_at_20 - value: 28.347 - - type: map_at_3 - value: 25.188 - - type: map_at_5 - value: 26.756999999999998 - - type: mrr_at_1 - value: 24.14012738853503 - - type: mrr_at_10 - value: 32.26827418865631 - - type: mrr_at_100 - value: 33.191905671640306 - - type: mrr_at_1000 - value: 33.25234496385887 - - type: mrr_at_20 - value: 32.83671423953111 - - type: mrr_at_3 - value: 29.893842887473426 - - type: mrr_at_5 - value: 31.419320594479768 - - type: nauc_map_at_1000_diff1 - value: 41.76617526166196 - - type: nauc_map_at_1000_max - value: 32.36105298409472 - - type: nauc_map_at_1000_std - value: 3.6495511496519155 - - type: nauc_map_at_100_diff1 - value: 41.78018349925877 - - type: nauc_map_at_100_max - value: 32.28108511470037 - - type: nauc_map_at_100_std - value: 3.5129991248899075 - - type: nauc_map_at_10_diff1 - value: 41.98486116566451 - - type: nauc_map_at_10_max - value: 32.047237898567715 - - type: nauc_map_at_10_std - value: 2.7440607363035925 - - type: nauc_map_at_1_diff1 - value: 47.14494297512505 - - type: nauc_map_at_1_max - value: 30.729904533725882 - - type: nauc_map_at_1_std - value: -1.4748123251677638 - - type: nauc_map_at_20_diff1 - value: 41.87357825935804 - - type: nauc_map_at_20_max - value: 32.074571884360616 - - type: nauc_map_at_20_std - value: 3.0650478802649506 - - type: nauc_map_at_3_diff1 - value: 43.44660277541162 - - type: nauc_map_at_3_max - value: 31.5736965905546 - - type: nauc_map_at_3_std - value: 1.2023760907565173 - - type: nauc_map_at_5_diff1 - value: 42.63344243739587 - - type: nauc_map_at_5_max - value: 31.67816525700815 - - type: nauc_map_at_5_std - value: 2.0332027700373905 - - type: nauc_mrr_at_1000_diff1 - value: 40.21948167058019 - - type: nauc_mrr_at_1000_max - value: 33.42753771790736 - - type: nauc_mrr_at_1000_std - value: 5.933854401399415 - - type: nauc_mrr_at_100_diff1 - value: 40.207921855075405 - - type: nauc_mrr_at_100_max - value: 33.40962655654419 - - type: nauc_mrr_at_100_std - value: 5.9276614649538395 - - type: nauc_mrr_at_10_diff1 - value: 40.24913874148852 - - type: nauc_mrr_at_10_max - value: 33.44670144717516 - - type: nauc_mrr_at_10_std - value: 5.676393545620746 - - type: nauc_mrr_at_1_diff1 - value: 46.38319251926657 - - type: nauc_mrr_at_1_max - value: 34.74359913713124 - - type: nauc_mrr_at_1_std - value: 3.082081961113807 - - type: nauc_mrr_at_20_diff1 - value: 40.126314085897654 - - type: nauc_mrr_at_20_max - value: 33.37213958773027 - - type: nauc_mrr_at_20_std - value: 5.861425354325102 - - type: nauc_mrr_at_3_diff1 - value: 41.76392677686872 - - type: nauc_mrr_at_3_max - value: 33.75783375631955 - - type: nauc_mrr_at_3_std - value: 5.242263033695973 - - type: nauc_mrr_at_5_diff1 - value: 40.66128066488706 - - type: nauc_mrr_at_5_max - value: 33.22364679237956 - - type: nauc_mrr_at_5_std - value: 5.430528733486765 - - type: nauc_ndcg_at_1000_diff1 - value: 38.63357657667927 - - type: nauc_ndcg_at_1000_max - value: 33.52315610562187 - - type: nauc_ndcg_at_1000_std - value: 8.432732805029033 - - type: nauc_ndcg_at_100_diff1 - value: 38.603550373052855 - - type: nauc_ndcg_at_100_max - value: 32.78553003015518 - - type: nauc_ndcg_at_100_std - value: 7.665142929815671 - - type: nauc_ndcg_at_10_diff1 - value: 39.17218005607024 - - type: nauc_ndcg_at_10_max - value: 32.37648122534756 - - type: nauc_ndcg_at_10_std - value: 5.229213836558875 - - type: nauc_ndcg_at_1_diff1 - value: 46.38319251926657 - - type: nauc_ndcg_at_1_max - value: 34.74359913713124 - - type: nauc_ndcg_at_1_std - value: 3.082081961113807 - - type: nauc_ndcg_at_20_diff1 - value: 38.839733199852226 - - type: nauc_ndcg_at_20_max - value: 32.16751688681974 - - type: nauc_ndcg_at_20_std - value: 5.9529585154962 - - type: nauc_ndcg_at_3_diff1 - value: 41.44525877654365 - - type: nauc_ndcg_at_3_max - value: 32.42707626231848 - - type: nauc_ndcg_at_3_std - value: 3.5522432042425214 - - type: nauc_ndcg_at_5_diff1 - value: 40.20334324643417 - - type: nauc_ndcg_at_5_max - value: 31.866704090360177 - - type: nauc_ndcg_at_5_std - value: 4.129311453116482 - - type: nauc_precision_at_1000_diff1 - value: -2.7079160999067655 - - type: nauc_precision_at_1000_max - value: 13.96843551636217 - - type: nauc_precision_at_1000_std - value: 22.67619361153759 - - type: nauc_precision_at_100_diff1 - value: 5.367108735367731 - - type: nauc_precision_at_100_max - value: 23.261316733202513 - - type: nauc_precision_at_100_std - value: 26.503281683469794 - - type: nauc_precision_at_10_diff1 - value: 21.861845852630704 - - type: nauc_precision_at_10_max - value: 33.035047051487695 - - type: nauc_precision_at_10_std - value: 17.695093243551263 - - type: nauc_precision_at_1_diff1 - value: 46.38319251926657 - - type: nauc_precision_at_1_max - value: 34.74359913713124 - - type: nauc_precision_at_1_std - value: 3.082081961113807 - - type: nauc_precision_at_20_diff1 - value: 16.086225605250913 - - type: nauc_precision_at_20_max - value: 29.111406274685685 - - type: nauc_precision_at_20_std - value: 20.16047627291658 - - type: nauc_precision_at_3_diff1 - value: 34.87770156593762 - - type: nauc_precision_at_3_max - value: 34.36598800372885 - - type: nauc_precision_at_3_std - value: 9.713422411448411 - - type: nauc_precision_at_5_diff1 - value: 28.556995540691215 - - type: nauc_precision_at_5_max - value: 32.41775704590351 - - type: nauc_precision_at_5_std - value: 12.526037082673245 - - type: nauc_recall_at_1000_diff1 - value: 22.518923633684242 - - type: nauc_recall_at_1000_max - value: 34.77756797992028 - - type: nauc_recall_at_1000_std - value: 30.96602342569516 - - type: nauc_recall_at_100_diff1 - value: 26.977344899465066 - - type: nauc_recall_at_100_max - value: 29.25577591297841 - - type: nauc_recall_at_100_std - value: 18.53329327689213 - - type: nauc_recall_at_10_diff1 - value: 31.833116167561254 - - type: nauc_recall_at_10_max - value: 29.53270517235027 - - type: nauc_recall_at_10_std - value: 7.5765043210495655 - - type: nauc_recall_at_1_diff1 - value: 47.14494297512505 - - type: nauc_recall_at_1_max - value: 30.729904533725882 - - type: nauc_recall_at_1_std - value: -1.4748123251677638 - - type: nauc_recall_at_20_diff1 - value: 29.522302969072467 - - type: nauc_recall_at_20_max - value: 27.881678480167693 - - type: nauc_recall_at_20_std - value: 10.0707944266602 - - type: nauc_recall_at_3_diff1 - value: 38.57308910390624 - - type: nauc_recall_at_3_max - value: 29.974741414471616 - - type: nauc_recall_at_3_std - value: 3.2396834673176493 - - type: nauc_recall_at_5_diff1 - value: 35.39828204400043 - - type: nauc_recall_at_5_max - value: 29.029908154639784 - - type: nauc_recall_at_5_std - value: 5.034024931108196 - - type: ndcg_at_1 - value: 24.14 - - type: ndcg_at_10 - value: 32.666000000000004 - - type: ndcg_at_100 - value: 37.734 - - type: ndcg_at_1000 - value: 40.511 - - type: ndcg_at_20 - value: 34.628 - - type: ndcg_at_3 - value: 28.509 - - type: ndcg_at_5 - value: 30.813000000000002 - - type: precision_at_1 - value: 24.14 - - type: precision_at_10 - value: 6.248 - - type: precision_at_100 - value: 1.113 - - type: precision_at_1000 - value: 0.169 - - type: precision_at_20 - value: 3.803 - - type: precision_at_3 - value: 13.736999999999998 - - type: precision_at_5 - value: 10.241999999999999 - - type: recall_at_1 - value: 19.496 - - type: recall_at_10 - value: 42.402 - - type: recall_at_100 - value: 64.252 - - type: recall_at_1000 - value: 82.32199999999999 - - type: recall_at_20 - value: 49.65 - - type: recall_at_3 - value: 30.927 - - type: recall_at_5 - value: 36.829 - task: - type: Retrieval - - dataset: - config: default - name: MTEB CQADupstackGamingRetrieval (default) - revision: 4885aa143210c98657558c04aaf3dc47cfb54340 - split: test - type: mteb/cqadupstack-gaming - metrics: - - type: main_score - value: 39.403 - - type: map_at_1 - value: 23.953 - - type: map_at_10 - value: 33.771 - - type: map_at_100 - value: 35.095 - - type: map_at_1000 - value: 35.184 - - type: map_at_20 - value: 34.531 - - type: map_at_3 - value: 30.705 - - type: map_at_5 - value: 32.468 - - type: mrr_at_1 - value: 27.711598746081506 - - type: mrr_at_10 - value: 36.88045479424789 - - type: mrr_at_100 - value: 37.97990481560263 - - type: mrr_at_1000 - value: 38.02816011291392 - - type: mrr_at_20 - value: 37.55076085643743 - - type: mrr_at_3 - value: 34.19017763845347 - - type: mrr_at_5 - value: 35.70741901776383 - - type: nauc_map_at_1000_diff1 - value: 36.00981920501309 - - type: nauc_map_at_1000_max - value: 22.095524568873014 - - type: nauc_map_at_1000_std - value: -2.8665621594465307 - - type: nauc_map_at_100_diff1 - value: 36.002011911243976 - - type: nauc_map_at_100_max - value: 22.104565543592557 - - type: nauc_map_at_100_std - value: -2.867602396981025 - - type: nauc_map_at_10_diff1 - value: 35.78951669531971 - - type: nauc_map_at_10_max - value: 21.454593639617812 - - type: nauc_map_at_10_std - value: -3.6974855600540453 - - type: nauc_map_at_1_diff1 - value: 40.53477828363302 - - type: nauc_map_at_1_max - value: 18.46790643899047 - - type: nauc_map_at_1_std - value: -7.22165065662934 - - type: nauc_map_at_20_diff1 - value: 36.010838525304834 - - type: nauc_map_at_20_max - value: 21.885046883123593 - - type: nauc_map_at_20_std - value: -3.210375937430325 - - type: nauc_map_at_3_diff1 - value: 37.418022634214516 - - type: nauc_map_at_3_max - value: 21.073669021006832 - - type: nauc_map_at_3_std - value: -4.679404664686778 - - type: nauc_map_at_5_diff1 - value: 36.6795123593822 - - type: nauc_map_at_5_max - value: 21.389015358073685 - - type: nauc_map_at_5_std - value: -4.325648408069168 - - type: nauc_mrr_at_1000_diff1 - value: 34.95611640945783 - - type: nauc_mrr_at_1000_max - value: 23.928491774707165 - - type: nauc_mrr_at_1000_std - value: -1.338791971396386 - - type: nauc_mrr_at_100_diff1 - value: 34.929193528403616 - - type: nauc_mrr_at_100_max - value: 23.939943348709594 - - type: nauc_mrr_at_100_std - value: -1.3082650041533628 - - type: nauc_mrr_at_10_diff1 - value: 34.69031733385043 - - type: nauc_mrr_at_10_max - value: 23.59274582738218 - - type: nauc_mrr_at_10_std - value: -1.8289688836101456 - - type: nauc_mrr_at_1_diff1 - value: 39.34734143208132 - - type: nauc_mrr_at_1_max - value: 21.50394761714022 - - type: nauc_mrr_at_1_std - value: -5.387089032695643 - - type: nauc_mrr_at_20_diff1 - value: 34.91303250313781 - - type: nauc_mrr_at_20_max - value: 23.847491555315674 - - type: nauc_mrr_at_20_std - value: -1.4304205162891497 - - type: nauc_mrr_at_3_diff1 - value: 36.18259864668598 - - type: nauc_mrr_at_3_max - value: 23.887811503907034 - - type: nauc_mrr_at_3_std - value: -2.464439729797598 - - type: nauc_mrr_at_5_diff1 - value: 35.33437531845849 - - type: nauc_mrr_at_5_max - value: 23.765452586249292 - - type: nauc_mrr_at_5_std - value: -2.1851855395426263 - - type: nauc_ndcg_at_1000_diff1 - value: 34.09418742679263 - - type: nauc_ndcg_at_1000_max - value: 24.55238871231702 - - type: nauc_ndcg_at_1000_std - value: 1.3417861971730751 - - type: nauc_ndcg_at_100_diff1 - value: 33.50605572479613 - - type: nauc_ndcg_at_100_max - value: 24.788837094333715 - - type: nauc_ndcg_at_100_std - value: 1.94905635328888 - - type: nauc_ndcg_at_10_diff1 - value: 32.99617129249672 - - type: nauc_ndcg_at_10_max - value: 22.373041946409547 - - type: nauc_ndcg_at_10_std - value: -1.6123251442731485 - - type: nauc_ndcg_at_1_diff1 - value: 39.34734143208132 - - type: nauc_ndcg_at_1_max - value: 21.50394761714022 - - type: nauc_ndcg_at_1_std - value: -5.387089032695643 - - type: nauc_ndcg_at_20_diff1 - value: 33.76153285165386 - - type: nauc_ndcg_at_20_max - value: 23.66281236035479 - - type: nauc_ndcg_at_20_std - value: 0.1314652340010358 - - type: nauc_ndcg_at_3_diff1 - value: 36.215628862018875 - - type: nauc_ndcg_at_3_max - value: 22.23728810221821 - - type: nauc_ndcg_at_3_std - value: -3.375324547297835 - - type: nauc_ndcg_at_5_diff1 - value: 34.869980250497065 - - type: nauc_ndcg_at_5_max - value: 22.375617254550175 - - type: nauc_ndcg_at_5_std - value: -2.8765947297571595 - - type: nauc_precision_at_1000_diff1 - value: -5.477133135940843 - - type: nauc_precision_at_1000_max - value: 11.714063664418656 - - type: nauc_precision_at_1000_std - value: 14.803511361321767 - - type: nauc_precision_at_100_diff1 - value: 1.190308506686791 - - type: nauc_precision_at_100_max - value: 20.879498285480025 - - type: nauc_precision_at_100_std - value: 19.864715556174225 - - type: nauc_precision_at_10_diff1 - value: 16.690689283041117 - - type: nauc_precision_at_10_max - value: 23.615626182492825 - - type: nauc_precision_at_10_std - value: 6.9604996221038755 - - type: nauc_precision_at_1_diff1 - value: 39.34734143208132 - - type: nauc_precision_at_1_max - value: 21.50394761714022 - - type: nauc_precision_at_1_std - value: -5.387089032695643 - - type: nauc_precision_at_20_diff1 - value: 13.941370133262929 - - type: nauc_precision_at_20_max - value: 25.0983395906178 - - type: nauc_precision_at_20_std - value: 12.863365250801706 - - type: nauc_precision_at_3_diff1 - value: 29.70706108250014 - - type: nauc_precision_at_3_max - value: 25.97857880896327 - - type: nauc_precision_at_3_std - value: 0.4671634667547243 - - type: nauc_precision_at_5_diff1 - value: 24.26712316402683 - - type: nauc_precision_at_5_max - value: 24.994467132277265 - - type: nauc_precision_at_5_std - value: 2.1020033045283952 - - type: nauc_recall_at_1000_diff1 - value: 21.036527285628885 - - type: nauc_recall_at_1000_max - value: 46.359454397435925 - - type: nauc_recall_at_1000_std - value: 47.68561830236934 - - type: nauc_recall_at_100_diff1 - value: 19.685774418991592 - - type: nauc_recall_at_100_max - value: 34.82581080838536 - - type: nauc_recall_at_100_std - value: 27.585463129717606 - - type: nauc_recall_at_10_diff1 - value: 23.74918228033508 - - type: nauc_recall_at_10_max - value: 21.23054148890896 - - type: nauc_recall_at_10_std - value: 2.5379471535492675 - - type: nauc_recall_at_1_diff1 - value: 40.53477828363302 - - type: nauc_recall_at_1_max - value: 18.46790643899047 - - type: nauc_recall_at_1_std - value: -7.22165065662934 - - type: nauc_recall_at_20_diff1 - value: 25.739859302267693 - - type: nauc_recall_at_20_max - value: 25.552244146602355 - - type: nauc_recall_at_20_std - value: 9.719380851403539 - - type: nauc_recall_at_3_diff1 - value: 33.44976234422277 - - type: nauc_recall_at_3_max - value: 22.10543969450477 - - type: nauc_recall_at_3_std - value: -1.7497643333208013 - - type: nauc_recall_at_5_diff1 - value: 29.95702802348527 - - type: nauc_recall_at_5_max - value: 21.90857504616491 - - type: nauc_recall_at_5_std - value: -0.6634375310704549 - - type: ndcg_at_1 - value: 27.711999999999996 - - type: ndcg_at_10 - value: 39.403 - - type: ndcg_at_100 - value: 45.327 - - type: ndcg_at_1000 - value: 47.221000000000004 - - type: ndcg_at_20 - value: 41.921 - - type: ndcg_at_3 - value: 33.657 - - type: ndcg_at_5 - value: 36.463 - - type: precision_at_1 - value: 27.711999999999996 - - type: precision_at_10 - value: 6.715 - - type: precision_at_100 - value: 1.068 - - type: precision_at_1000 - value: 0.13 - - type: precision_at_20 - value: 4.034 - - type: precision_at_3 - value: 15.360999999999999 - - type: precision_at_5 - value: 11.06 - - type: recall_at_1 - value: 23.953 - - type: recall_at_10 - value: 53.554 - - type: recall_at_100 - value: 79.39699999999999 - - type: recall_at_1000 - value: 92.85900000000001 - - type: recall_at_20 - value: 62.90599999999999 - - type: recall_at_3 - value: 38.074000000000005 - - type: recall_at_5 - value: 44.869 - task: - type: Retrieval - - dataset: - config: default - name: MTEB CQADupstackGisRetrieval (default) - revision: 5003b3064772da1887988e05400cf3806fe491f2 - split: test - type: mteb/cqadupstack-gis - metrics: - - type: main_score - value: 20.723 - - type: map_at_1 - value: 11.454 - - type: map_at_10 - value: 16.933 - - type: map_at_100 - value: 17.931 - - type: map_at_1000 - value: 18.066 - - type: map_at_20 - value: 17.444000000000003 - - type: map_at_3 - value: 14.821000000000002 - - type: map_at_5 - value: 15.847 - - type: mrr_at_1 - value: 12.88135593220339 - - type: mrr_at_10 - value: 18.568648551699386 - - type: mrr_at_100 - value: 19.519728948766964 - - type: mrr_at_1000 - value: 19.629818718309142 - - type: mrr_at_20 - value: 19.04251586406436 - - type: mrr_at_3 - value: 16.38418079096045 - - type: mrr_at_5 - value: 17.446327683615817 - - type: nauc_map_at_1000_diff1 - value: 31.00971211104956 - - type: nauc_map_at_1000_max - value: 13.850017784244987 - - type: nauc_map_at_1000_std - value: 8.963659587175178 - - type: nauc_map_at_100_diff1 - value: 30.974620576122003 - - type: nauc_map_at_100_max - value: 13.853314356539528 - - type: nauc_map_at_100_std - value: 8.954247164461094 - - type: nauc_map_at_10_diff1 - value: 31.109975868798433 - - type: nauc_map_at_10_max - value: 13.51348567087792 - - type: nauc_map_at_10_std - value: 8.550888012900435 - - type: nauc_map_at_1_diff1 - value: 39.84333952813111 - - type: nauc_map_at_1_max - value: 16.068746301688737 - - type: nauc_map_at_1_std - value: 4.093189268230556 - - type: nauc_map_at_20_diff1 - value: 30.85752130769573 - - type: nauc_map_at_20_max - value: 13.63962555746682 - - type: nauc_map_at_20_std - value: 8.709432284644775 - - type: nauc_map_at_3_diff1 - value: 32.80973726691313 - - type: nauc_map_at_3_max - value: 14.284720966613204 - - type: nauc_map_at_3_std - value: 6.1608525290902385 - - type: nauc_map_at_5_diff1 - value: 31.808730696207004 - - type: nauc_map_at_5_max - value: 13.834242310684614 - - type: nauc_map_at_5_std - value: 6.835723601421637 - - type: nauc_mrr_at_1000_diff1 - value: 29.759046423357933 - - type: nauc_mrr_at_1000_max - value: 14.951338971704445 - - type: nauc_mrr_at_1000_std - value: 10.113349487019041 - - type: nauc_mrr_at_100_diff1 - value: 29.71206355838227 - - type: nauc_mrr_at_100_max - value: 14.9588343997006 - - type: nauc_mrr_at_100_std - value: 10.147966222836807 - - type: nauc_mrr_at_10_diff1 - value: 29.626989847475798 - - type: nauc_mrr_at_10_max - value: 14.74042625656872 - - type: nauc_mrr_at_10_std - value: 9.839974138708314 - - type: nauc_mrr_at_1_diff1 - value: 38.38214921179599 - - type: nauc_mrr_at_1_max - value: 16.904444938773945 - - type: nauc_mrr_at_1_std - value: 5.461433614362911 - - type: nauc_mrr_at_20_diff1 - value: 29.582744417399258 - - type: nauc_mrr_at_20_max - value: 14.873525526797792 - - type: nauc_mrr_at_20_std - value: 10.02659346294331 - - type: nauc_mrr_at_3_diff1 - value: 31.3488783212548 - - type: nauc_mrr_at_3_max - value: 15.483058306281643 - - type: nauc_mrr_at_3_std - value: 8.200409977944425 - - type: nauc_mrr_at_5_diff1 - value: 30.16593682015114 - - type: nauc_mrr_at_5_max - value: 14.859820711531091 - - type: nauc_mrr_at_5_std - value: 8.185908135154364 - - type: nauc_ndcg_at_1000_diff1 - value: 28.18547695388205 - - type: nauc_ndcg_at_1000_max - value: 14.005327206216434 - - type: nauc_ndcg_at_1000_std - value: 12.589669560088968 - - type: nauc_ndcg_at_100_diff1 - value: 27.758324300357984 - - type: nauc_ndcg_at_100_max - value: 14.341133555972089 - - type: nauc_ndcg_at_100_std - value: 13.31250167325408 - - type: nauc_ndcg_at_10_diff1 - value: 27.905796760757134 - - type: nauc_ndcg_at_10_max - value: 12.96750126038255 - - type: nauc_ndcg_at_10_std - value: 11.252396272365992 - - type: nauc_ndcg_at_1_diff1 - value: 38.38214921179599 - - type: nauc_ndcg_at_1_max - value: 16.904444938773945 - - type: nauc_ndcg_at_1_std - value: 5.461433614362911 - - type: nauc_ndcg_at_20_diff1 - value: 27.349876344089957 - - type: nauc_ndcg_at_20_max - value: 13.320914532200256 - - type: nauc_ndcg_at_20_std - value: 11.867410167033551 - - type: nauc_ndcg_at_3_diff1 - value: 30.343259237299268 - - type: nauc_ndcg_at_3_max - value: 14.201276650661434 - - type: nauc_ndcg_at_3_std - value: 7.12523611875583 - - type: nauc_ndcg_at_5_diff1 - value: 28.801943226027753 - - type: nauc_ndcg_at_5_max - value: 13.338551217810954 - - type: nauc_ndcg_at_5_std - value: 7.801128464664271 - - type: nauc_precision_at_1000_diff1 - value: 2.001268152281262 - - type: nauc_precision_at_1000_max - value: 10.612211087722672 - - type: nauc_precision_at_1000_std - value: 18.232225350370506 - - type: nauc_precision_at_100_diff1 - value: 12.999552197478135 - - type: nauc_precision_at_100_max - value: 16.33859747515745 - - type: nauc_precision_at_100_std - value: 26.45969466756656 - - type: nauc_precision_at_10_diff1 - value: 18.117672151473986 - - type: nauc_precision_at_10_max - value: 12.306841727543809 - - type: nauc_precision_at_10_std - value: 19.224817620570555 - - type: nauc_precision_at_1_diff1 - value: 38.38214921179599 - - type: nauc_precision_at_1_max - value: 16.904444938773945 - - type: nauc_precision_at_1_std - value: 5.461433614362911 - - type: nauc_precision_at_20_diff1 - value: 15.115529221920223 - - type: nauc_precision_at_20_max - value: 12.89077612414914 - - type: nauc_precision_at_20_std - value: 20.414316524315318 - - type: nauc_precision_at_3_diff1 - value: 25.03722914022995 - - type: nauc_precision_at_3_max - value: 15.709378585870356 - - type: nauc_precision_at_3_std - value: 10.010187249834738 - - type: nauc_precision_at_5_diff1 - value: 21.525625871418207 - - type: nauc_precision_at_5_max - value: 13.497803642649473 - - type: nauc_precision_at_5_std - value: 11.075155507096765 - - type: nauc_recall_at_1000_diff1 - value: 18.280987536015267 - - type: nauc_recall_at_1000_max - value: 9.574683101626952 - - type: nauc_recall_at_1000_std - value: 23.448036040224913 - - type: nauc_recall_at_100_diff1 - value: 20.578493216367274 - - type: nauc_recall_at_100_max - value: 14.56962816210217 - - type: nauc_recall_at_100_std - value: 23.41963721799639 - - type: nauc_recall_at_10_diff1 - value: 21.583257614269808 - - type: nauc_recall_at_10_max - value: 10.242354988584784 - - type: nauc_recall_at_10_std - value: 15.98063513181825 - - type: nauc_recall_at_1_diff1 - value: 39.84333952813111 - - type: nauc_recall_at_1_max - value: 16.068746301688737 - - type: nauc_recall_at_1_std - value: 4.093189268230556 - - type: nauc_recall_at_20_diff1 - value: 20.352337875547352 - - type: nauc_recall_at_20_max - value: 11.292402190116817 - - type: nauc_recall_at_20_std - value: 17.604505218199172 - - type: nauc_recall_at_3_diff1 - value: 26.04480028663088 - - type: nauc_recall_at_3_max - value: 12.75221283182816 - - type: nauc_recall_at_3_std - value: 7.771755623504322 - - type: nauc_recall_at_5_diff1 - value: 22.67971113828535 - - type: nauc_recall_at_5_max - value: 10.998067327387378 - - type: nauc_recall_at_5_std - value: 8.733032345555438 - - type: ndcg_at_1 - value: 12.881 - - type: ndcg_at_10 - value: 20.723 - - type: ndcg_at_100 - value: 26.068 - - type: ndcg_at_1000 - value: 29.593999999999998 - - type: ndcg_at_20 - value: 22.462 - - type: ndcg_at_3 - value: 16.351 - - type: ndcg_at_5 - value: 18.119 - - type: precision_at_1 - value: 12.881 - - type: precision_at_10 - value: 3.6380000000000003 - - type: precision_at_100 - value: 0.675 - - type: precision_at_1000 - value: 0.10200000000000001 - - type: precision_at_20 - value: 2.22 - - type: precision_at_3 - value: 7.269 - - type: precision_at_5 - value: 5.379 - - type: recall_at_1 - value: 11.454 - - type: recall_at_10 - value: 31.119000000000003 - - type: recall_at_100 - value: 56.64900000000001 - - type: recall_at_1000 - value: 83.599 - - type: recall_at_20 - value: 37.657000000000004 - - type: recall_at_3 - value: 19.035 - - type: recall_at_5 - value: 23.307 - task: - type: Retrieval - - dataset: - config: default - name: MTEB CQADupstackMathematicaRetrieval (default) - revision: 90fceea13679c63fe563ded68f3b6f06e50061de - split: test - type: mteb/cqadupstack-mathematica - metrics: - - type: main_score - value: 18.602 - - type: map_at_1 - value: 8.233 - - type: map_at_10 - value: 14.263 - - type: map_at_100 - value: 15.634999999999998 - - type: map_at_1000 - value: 15.781999999999998 - - type: map_at_20 - value: 14.959 - - type: map_at_3 - value: 11.889 - - type: map_at_5 - value: 13.067 - - type: mrr_at_1 - value: 9.950248756218906 - - type: mrr_at_10 - value: 16.84430032377793 - - type: mrr_at_100 - value: 18.059357585498777 - - type: mrr_at_1000 - value: 18.154524917526423 - - type: mrr_at_20 - value: 17.474440783148786 - - type: mrr_at_3 - value: 14.510779436152562 - - type: mrr_at_5 - value: 15.667495854063018 - - type: nauc_map_at_1000_diff1 - value: 18.23217746190047 - - type: nauc_map_at_1000_max - value: 20.49022753078953 - - type: nauc_map_at_1000_std - value: 14.846867678528303 - - type: nauc_map_at_100_diff1 - value: 18.222951182346726 - - type: nauc_map_at_100_max - value: 20.462428579695047 - - type: nauc_map_at_100_std - value: 14.801451610995041 - - type: nauc_map_at_10_diff1 - value: 18.806998168246512 - - type: nauc_map_at_10_max - value: 20.346327683018394 - - type: nauc_map_at_10_std - value: 14.781486965673698 - - type: nauc_map_at_1_diff1 - value: 24.90271842269989 - - type: nauc_map_at_1_max - value: 20.605821895768834 - - type: nauc_map_at_1_std - value: 13.654354962612034 - - type: nauc_map_at_20_diff1 - value: 18.30322394754611 - - type: nauc_map_at_20_max - value: 20.25406227052277 - - type: nauc_map_at_20_std - value: 14.33873197584387 - - type: nauc_map_at_3_diff1 - value: 20.17446716046726 - - type: nauc_map_at_3_max - value: 19.11985380269457 - - type: nauc_map_at_3_std - value: 12.503935147559375 - - type: nauc_map_at_5_diff1 - value: 18.687432760432813 - - type: nauc_map_at_5_max - value: 19.87068913715553 - - type: nauc_map_at_5_std - value: 14.10119379199484 - - type: nauc_mrr_at_1000_diff1 - value: 15.686926847621882 - - type: nauc_mrr_at_1000_max - value: 20.41838504194867 - - type: nauc_mrr_at_1000_std - value: 14.601678542982032 - - type: nauc_mrr_at_100_diff1 - value: 15.649148415618786 - - type: nauc_mrr_at_100_max - value: 20.390471061201453 - - type: nauc_mrr_at_100_std - value: 14.570935730839452 - - type: nauc_mrr_at_10_diff1 - value: 15.799150500411855 - - type: nauc_mrr_at_10_max - value: 20.575744858600387 - - type: nauc_mrr_at_10_std - value: 14.768268932423927 - - type: nauc_mrr_at_1_diff1 - value: 23.367444138495525 - - type: nauc_mrr_at_1_max - value: 20.695137503965665 - - type: nauc_mrr_at_1_std - value: 14.72157770136156 - - type: nauc_mrr_at_20_diff1 - value: 15.69282182722306 - - type: nauc_mrr_at_20_max - value: 20.496725287008648 - - type: nauc_mrr_at_20_std - value: 14.388349301754728 - - type: nauc_mrr_at_3_diff1 - value: 16.552667983195963 - - type: nauc_mrr_at_3_max - value: 19.30089479846865 - - type: nauc_mrr_at_3_std - value: 12.419365827975918 - - type: nauc_mrr_at_5_diff1 - value: 15.2021345488141 - - type: nauc_mrr_at_5_max - value: 19.863155583605863 - - type: nauc_mrr_at_5_std - value: 13.981416058831526 - - type: nauc_ndcg_at_1000_diff1 - value: 15.496613143969418 - - type: nauc_ndcg_at_1000_max - value: 22.093806024787852 - - type: nauc_ndcg_at_1000_std - value: 17.766995355844745 - - type: nauc_ndcg_at_100_diff1 - value: 15.256710001281315 - - type: nauc_ndcg_at_100_max - value: 21.85870766599504 - - type: nauc_ndcg_at_100_std - value: 17.06357990658541 - - type: nauc_ndcg_at_10_diff1 - value: 16.645396625915108 - - type: nauc_ndcg_at_10_max - value: 21.533816489312795 - - type: nauc_ndcg_at_10_std - value: 16.29564024579396 - - type: nauc_ndcg_at_1_diff1 - value: 23.367444138495525 - - type: nauc_ndcg_at_1_max - value: 20.695137503965665 - - type: nauc_ndcg_at_1_std - value: 14.72157770136156 - - type: nauc_ndcg_at_20_diff1 - value: 15.5570017125561 - - type: nauc_ndcg_at_20_max - value: 21.359185072455986 - - type: nauc_ndcg_at_20_std - value: 15.057162550772377 - - type: nauc_ndcg_at_3_diff1 - value: 18.148078513804155 - - type: nauc_ndcg_at_3_max - value: 19.413219163942884 - - type: nauc_ndcg_at_3_std - value: 12.206859061209872 - - type: nauc_ndcg_at_5_diff1 - value: 15.946387582108482 - - type: nauc_ndcg_at_5_max - value: 20.614438981415713 - - type: nauc_ndcg_at_5_std - value: 14.956938588009589 - - type: nauc_precision_at_1000_diff1 - value: -6.338405284332406 - - type: nauc_precision_at_1000_max - value: 6.367574020314142 - - type: nauc_precision_at_1000_std - value: 3.2948111384908634 - - type: nauc_precision_at_100_diff1 - value: 2.5963271076466286 - - type: nauc_precision_at_100_max - value: 17.195718920975406 - - type: nauc_precision_at_100_std - value: 13.44480018562989 - - type: nauc_precision_at_10_diff1 - value: 9.894164875058406 - - type: nauc_precision_at_10_max - value: 22.607176912959513 - - type: nauc_precision_at_10_std - value: 16.602991509521992 - - type: nauc_precision_at_1_diff1 - value: 23.367444138495525 - - type: nauc_precision_at_1_max - value: 20.695137503965665 - - type: nauc_precision_at_1_std - value: 14.72157770136156 - - type: nauc_precision_at_20_diff1 - value: 7.479617555395287 - - type: nauc_precision_at_20_max - value: 20.646048773417565 - - type: nauc_precision_at_20_std - value: 12.587065342832732 - - type: nauc_precision_at_3_diff1 - value: 13.549141346398994 - - type: nauc_precision_at_3_max - value: 19.096896253669218 - - type: nauc_precision_at_3_std - value: 10.98636215569535 - - type: nauc_precision_at_5_diff1 - value: 9.308854693332993 - - type: nauc_precision_at_5_max - value: 20.614038596547594 - - type: nauc_precision_at_5_std - value: 14.519366227680033 - - type: nauc_recall_at_1000_diff1 - value: 11.6542815361156 - - type: nauc_recall_at_1000_max - value: 32.692947390599 - - type: nauc_recall_at_1000_std - value: 44.65436374659935 - - type: nauc_recall_at_100_diff1 - value: 10.060501930914468 - - type: nauc_recall_at_100_max - value: 23.909265280968484 - - type: nauc_recall_at_100_std - value: 23.0879660121024 - - type: nauc_recall_at_10_diff1 - value: 14.302911670798506 - - type: nauc_recall_at_10_max - value: 22.85869910411015 - - type: nauc_recall_at_10_std - value: 18.963781625147185 - - type: nauc_recall_at_1_diff1 - value: 24.90271842269989 - - type: nauc_recall_at_1_max - value: 20.605821895768834 - - type: nauc_recall_at_1_std - value: 13.654354962612034 - - type: nauc_recall_at_20_diff1 - value: 11.411188213614896 - - type: nauc_recall_at_20_max - value: 22.259549408442762 - - type: nauc_recall_at_20_std - value: 15.507984814176625 - - type: nauc_recall_at_3_diff1 - value: 16.26909180904135 - - type: nauc_recall_at_3_max - value: 18.870658327598097 - - type: nauc_recall_at_3_std - value: 11.838400434931915 - - type: nauc_recall_at_5_diff1 - value: 12.244714380165343 - - type: nauc_recall_at_5_max - value: 21.261392668644532 - - type: nauc_recall_at_5_std - value: 16.651686356883157 - - type: ndcg_at_1 - value: 9.950000000000001 - - type: ndcg_at_10 - value: 18.602 - - type: ndcg_at_100 - value: 25.512 - - type: ndcg_at_1000 - value: 29.26 - - type: ndcg_at_20 - value: 21.02 - - type: ndcg_at_3 - value: 13.972000000000001 - - type: ndcg_at_5 - value: 15.876999999999999 - - type: precision_at_1 - value: 9.950000000000001 - - type: precision_at_10 - value: 3.943 - - type: precision_at_100 - value: 0.872 - - type: precision_at_1000 - value: 0.136 - - type: precision_at_20 - value: 2.643 - - type: precision_at_3 - value: 7.255000000000001 - - type: precision_at_5 - value: 5.622 - - type: recall_at_1 - value: 8.233 - - type: recall_at_10 - value: 29.366999999999997 - - type: recall_at_100 - value: 60.209 - - type: recall_at_1000 - value: 87.41499999999999 - - type: recall_at_20 - value: 38.190000000000005 - - type: recall_at_3 - value: 16.519000000000002 - - type: recall_at_5 - value: 21.319 - task: - type: Retrieval - - dataset: - config: default - name: MTEB CQADupstackPhysicsRetrieval (default) - revision: 79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4 - split: test - type: mteb/cqadupstack-physics - metrics: - - type: main_score - value: 32.592 - - type: map_at_1 - value: 18.758 - - type: map_at_10 - value: 27.198 - - type: map_at_100 - value: 28.706 - - type: map_at_1000 - value: 28.838 - - type: map_at_20 - value: 28.033 - - type: map_at_3 - value: 24.352999999999998 - - type: map_at_5 - value: 25.971 - - type: mrr_at_1 - value: 24.254090471607313 - - type: mrr_at_10 - value: 32.29337580396289 - - type: mrr_at_100 - value: 33.33713398298678 - - type: mrr_at_1000 - value: 33.40927242447723 - - type: mrr_at_20 - value: 32.86529042825282 - - type: mrr_at_3 - value: 29.77221687520053 - - type: mrr_at_5 - value: 31.158164902149494 - - type: nauc_map_at_1000_diff1 - value: 35.43332186007521 - - type: nauc_map_at_1000_max - value: 28.979622893773104 - - type: nauc_map_at_1000_std - value: 6.578956825302608 - - type: nauc_map_at_100_diff1 - value: 35.39022158661449 - - type: nauc_map_at_100_max - value: 28.94560731774703 - - type: nauc_map_at_100_std - value: 6.538195339460332 - - type: nauc_map_at_10_diff1 - value: 35.677884861017915 - - type: nauc_map_at_10_max - value: 28.57489475080354 - - type: nauc_map_at_10_std - value: 5.454103100616544 - - type: nauc_map_at_1_diff1 - value: 42.53458088369024 - - type: nauc_map_at_1_max - value: 25.886547599537934 - - type: nauc_map_at_1_std - value: 0.11639371074674959 - - type: nauc_map_at_20_diff1 - value: 35.48823910499732 - - type: nauc_map_at_20_max - value: 28.661222493545953 - - type: nauc_map_at_20_std - value: 6.0736999593558725 - - type: nauc_map_at_3_diff1 - value: 37.010247704093615 - - type: nauc_map_at_3_max - value: 27.961812044925672 - - type: nauc_map_at_3_std - value: 4.451362489929877 - - type: nauc_map_at_5_diff1 - value: 36.69428263148319 - - type: nauc_map_at_5_max - value: 27.802098249579487 - - type: nauc_map_at_5_std - value: 4.297801872956301 - - type: nauc_mrr_at_1000_diff1 - value: 34.559056308808934 - - type: nauc_mrr_at_1000_max - value: 32.83666199871945 - - type: nauc_mrr_at_1000_std - value: 11.867050560410966 - - type: nauc_mrr_at_100_diff1 - value: 34.523273474153534 - - type: nauc_mrr_at_100_max - value: 32.82393565429814 - - type: nauc_mrr_at_100_std - value: 11.866910023146005 - - type: nauc_mrr_at_10_diff1 - value: 34.532086131009926 - - type: nauc_mrr_at_10_max - value: 32.85146332978944 - - type: nauc_mrr_at_10_std - value: 11.71169987538628 - - type: nauc_mrr_at_1_diff1 - value: 39.335582470379094 - - type: nauc_mrr_at_1_max - value: 31.740439567406035 - - type: nauc_mrr_at_1_std - value: 8.529530277559985 - - type: nauc_mrr_at_20_diff1 - value: 34.54413479234956 - - type: nauc_mrr_at_20_max - value: 32.80857217884902 - - type: nauc_mrr_at_20_std - value: 11.817454680274084 - - type: nauc_mrr_at_3_diff1 - value: 35.72764069738432 - - type: nauc_mrr_at_3_max - value: 33.04686562750682 - - type: nauc_mrr_at_3_std - value: 11.45610782490127 - - type: nauc_mrr_at_5_diff1 - value: 35.339427295466805 - - type: nauc_mrr_at_5_max - value: 32.477167711422986 - - type: nauc_mrr_at_5_std - value: 10.970511640489987 - - type: nauc_ndcg_at_1000_diff1 - value: 32.972563023860495 - - type: nauc_ndcg_at_1000_max - value: 31.56124332885808 - - type: nauc_ndcg_at_1000_std - value: 11.320625017933688 - - type: nauc_ndcg_at_100_diff1 - value: 31.981617985898936 - - type: nauc_ndcg_at_100_max - value: 30.817448507218447 - - type: nauc_ndcg_at_100_std - value: 11.05131431144915 - - type: nauc_ndcg_at_10_diff1 - value: 32.898755384329704 - - type: nauc_ndcg_at_10_max - value: 29.83589745108241 - - type: nauc_ndcg_at_10_std - value: 8.082772747459629 - - type: nauc_ndcg_at_1_diff1 - value: 39.335582470379094 - - type: nauc_ndcg_at_1_max - value: 31.740439567406035 - - type: nauc_ndcg_at_1_std - value: 8.529530277559985 - - type: nauc_ndcg_at_20_diff1 - value: 32.412206781494795 - - type: nauc_ndcg_at_20_max - value: 29.861450129085316 - - type: nauc_ndcg_at_20_std - value: 9.383718452743128 - - type: nauc_ndcg_at_3_diff1 - value: 34.984136050599965 - - type: nauc_ndcg_at_3_max - value: 30.14318309067973 - - type: nauc_ndcg_at_3_std - value: 7.909917756441661 - - type: nauc_ndcg_at_5_diff1 - value: 34.92814489705384 - - type: nauc_ndcg_at_5_max - value: 29.16608661742856 - - type: nauc_ndcg_at_5_std - value: 6.694550269553597 - - type: nauc_precision_at_1000_diff1 - value: -3.790848810395332 - - type: nauc_precision_at_1000_max - value: 14.802421936399268 - - type: nauc_precision_at_1000_std - value: 23.145894849144554 - - type: nauc_precision_at_100_diff1 - value: 1.7546038039919793 - - type: nauc_precision_at_100_max - value: 25.662733100762296 - - type: nauc_precision_at_100_std - value: 31.225277020769827 - - type: nauc_precision_at_10_diff1 - value: 15.694416287771043 - - type: nauc_precision_at_10_max - value: 32.656449707547104 - - type: nauc_precision_at_10_std - value: 22.340228469453972 - - type: nauc_precision_at_1_diff1 - value: 39.335582470379094 - - type: nauc_precision_at_1_max - value: 31.740439567406035 - - type: nauc_precision_at_1_std - value: 8.529530277559985 - - type: nauc_precision_at_20_diff1 - value: 10.453000063206943 - - type: nauc_precision_at_20_max - value: 29.05806853523417 - - type: nauc_precision_at_20_std - value: 26.277225739049488 - - type: nauc_precision_at_3_diff1 - value: 27.272188733861135 - - type: nauc_precision_at_3_max - value: 34.83933062214277 - - type: nauc_precision_at_3_std - value: 17.775294609166153 - - type: nauc_precision_at_5_diff1 - value: 24.258907574802578 - - type: nauc_precision_at_5_max - value: 32.11919091242911 - - type: nauc_precision_at_5_std - value: 16.252625454457664 - - type: nauc_recall_at_1000_diff1 - value: 14.954086634753432 - - type: nauc_recall_at_1000_max - value: 42.80424989945532 - - type: nauc_recall_at_1000_std - value: 36.79872372413446 - - type: nauc_recall_at_100_diff1 - value: 16.048538619104164 - - type: nauc_recall_at_100_max - value: 27.368927292992325 - - type: nauc_recall_at_100_std - value: 18.887258365172094 - - type: nauc_recall_at_10_diff1 - value: 24.46647338879468 - - type: nauc_recall_at_10_max - value: 26.603563703221095 - - type: nauc_recall_at_10_std - value: 8.028133219751867 - - type: nauc_recall_at_1_diff1 - value: 42.53458088369024 - - type: nauc_recall_at_1_max - value: 25.886547599537934 - - type: nauc_recall_at_1_std - value: 0.11639371074674959 - - type: nauc_recall_at_20_diff1 - value: 22.40840509718229 - - type: nauc_recall_at_20_max - value: 25.805462126399785 - - type: nauc_recall_at_20_std - value: 11.278990263026085 - - type: nauc_recall_at_3_diff1 - value: 31.262823269379076 - - type: nauc_recall_at_3_max - value: 27.044002473477065 - - type: nauc_recall_at_3_std - value: 5.89017119130073 - - type: nauc_recall_at_5_diff1 - value: 30.18775263414265 - - type: nauc_recall_at_5_max - value: 25.583478864235904 - - type: nauc_recall_at_5_std - value: 4.569298948312717 - - type: ndcg_at_1 - value: 24.254 - - type: ndcg_at_10 - value: 32.592 - - type: ndcg_at_100 - value: 39.334 - - type: ndcg_at_1000 - value: 42.144999999999996 - - type: ndcg_at_20 - value: 35.195 - - type: ndcg_at_3 - value: 27.742 - - type: ndcg_at_5 - value: 30.035 - - type: precision_at_1 - value: 24.254 - - type: precision_at_10 - value: 6.381 - - type: precision_at_100 - value: 1.149 - - type: precision_at_1000 - value: 0.158 - - type: precision_at_20 - value: 3.9890000000000003 - - type: precision_at_3 - value: 13.474 - - type: precision_at_5 - value: 9.933 - - type: recall_at_1 - value: 18.758 - - type: recall_at_10 - value: 43.732 - - type: recall_at_100 - value: 73.152 - - type: recall_at_1000 - value: 92.08800000000001 - - type: recall_at_20 - value: 53.056000000000004 - - type: recall_at_3 - value: 30.348000000000003 - - type: recall_at_5 - value: 36.248000000000005 - task: - type: Retrieval - - dataset: - config: default - name: MTEB CQADupstackProgrammersRetrieval (default) - revision: 6184bc1440d2dbc7612be22b50686b8826d22b32 - split: test - type: mteb/cqadupstack-programmers - metrics: - - type: main_score - value: 27.023999999999997 - - type: map_at_1 - value: 14.198 - - type: map_at_10 - value: 21.75 - - type: map_at_100 - value: 23.328 - - type: map_at_1000 - value: 23.47 - - type: map_at_20 - value: 22.595000000000002 - - type: map_at_3 - value: 18.9 - - type: map_at_5 - value: 20.362 - - type: mrr_at_1 - value: 18.15068493150685 - - type: mrr_at_10 - value: 26.040262375878804 - - type: mrr_at_100 - value: 27.198199648239772 - - type: mrr_at_1000 - value: 27.27774611735573 - - type: mrr_at_20 - value: 26.691941189083767 - - type: mrr_at_3 - value: 23.173515981735168 - - type: mrr_at_5 - value: 24.686073059360726 - - type: nauc_map_at_1000_diff1 - value: 31.053453200715325 - - type: nauc_map_at_1000_max - value: 29.024393074756617 - - type: nauc_map_at_1000_std - value: 6.325913499941615 - - type: nauc_map_at_100_diff1 - value: 30.97303489032699 - - type: nauc_map_at_100_max - value: 29.029878260416496 - - type: nauc_map_at_100_std - value: 6.347144684839333 - - type: nauc_map_at_10_diff1 - value: 31.00470717224258 - - type: nauc_map_at_10_max - value: 28.019036460754265 - - type: nauc_map_at_10_std - value: 4.8665094208266275 - - type: nauc_map_at_1_diff1 - value: 37.47367952280333 - - type: nauc_map_at_1_max - value: 28.526104067645214 - - type: nauc_map_at_1_std - value: 0.42944400837957647 - - type: nauc_map_at_20_diff1 - value: 30.853722315305387 - - type: nauc_map_at_20_max - value: 28.37468657125153 - - type: nauc_map_at_20_std - value: 5.466774304327295 - - type: nauc_map_at_3_diff1 - value: 32.22525292125427 - - type: nauc_map_at_3_max - value: 27.100397776424597 - - type: nauc_map_at_3_std - value: 3.6780889748702723 - - type: nauc_map_at_5_diff1 - value: 30.72216748439938 - - type: nauc_map_at_5_max - value: 27.35756142039516 - - type: nauc_map_at_5_std - value: 4.68454886538082 - - type: nauc_mrr_at_1000_diff1 - value: 29.037948385015405 - - type: nauc_mrr_at_1000_max - value: 28.763423733914458 - - type: nauc_mrr_at_1000_std - value: 7.956994665321418 - - type: nauc_mrr_at_100_diff1 - value: 29.006585514645323 - - type: nauc_mrr_at_100_max - value: 28.789375493332596 - - type: nauc_mrr_at_100_std - value: 8.011211311161109 - - type: nauc_mrr_at_10_diff1 - value: 29.139613464995 - - type: nauc_mrr_at_10_max - value: 28.581685095094265 - - type: nauc_mrr_at_10_std - value: 7.6145044569554825 - - type: nauc_mrr_at_1_diff1 - value: 33.63214806586087 - - type: nauc_mrr_at_1_max - value: 28.809395842946238 - - type: nauc_mrr_at_1_std - value: 3.870367319917128 - - type: nauc_mrr_at_20_diff1 - value: 28.897527412848618 - - type: nauc_mrr_at_20_max - value: 28.44699870228015 - - type: nauc_mrr_at_20_std - value: 7.611077545742706 - - type: nauc_mrr_at_3_diff1 - value: 30.195365305410853 - - type: nauc_mrr_at_3_max - value: 27.875075763225105 - - type: nauc_mrr_at_3_std - value: 6.392092075078199 - - type: nauc_mrr_at_5_diff1 - value: 29.02331736193772 - - type: nauc_mrr_at_5_max - value: 28.526895681441516 - - type: nauc_mrr_at_5_std - value: 7.505926748987359 - - type: nauc_ndcg_at_1000_diff1 - value: 29.495231262556782 - - type: nauc_ndcg_at_1000_max - value: 31.136369251324425 - - type: nauc_ndcg_at_1000_std - value: 11.458769315946412 - - type: nauc_ndcg_at_100_diff1 - value: 28.14154383832849 - - type: nauc_ndcg_at_100_max - value: 31.317245100158054 - - type: nauc_ndcg_at_100_std - value: 12.343874658257208 - - type: nauc_ndcg_at_10_diff1 - value: 28.551238661045275 - - type: nauc_ndcg_at_10_max - value: 28.198394288716493 - - type: nauc_ndcg_at_10_std - value: 6.996000683674131 - - type: nauc_ndcg_at_1_diff1 - value: 33.63214806586087 - - type: nauc_ndcg_at_1_max - value: 28.809395842946238 - - type: nauc_ndcg_at_1_std - value: 3.870367319917128 - - type: nauc_ndcg_at_20_diff1 - value: 28.06030705119219 - - type: nauc_ndcg_at_20_max - value: 28.46993596667669 - - type: nauc_ndcg_at_20_std - value: 7.854692180311396 - - type: nauc_ndcg_at_3_diff1 - value: 30.333015112698174 - - type: nauc_ndcg_at_3_max - value: 27.250406313613716 - - type: nauc_ndcg_at_3_std - value: 5.627330922520453 - - type: nauc_ndcg_at_5_diff1 - value: 28.450912289686897 - - type: nauc_ndcg_at_5_max - value: 27.756166296817604 - - type: nauc_ndcg_at_5_std - value: 7.180600634098227 - - type: nauc_precision_at_1000_diff1 - value: 5.475615121974908 - - type: nauc_precision_at_1000_max - value: 12.647020683088266 - - type: nauc_precision_at_1000_std - value: 13.061720484832934 - - type: nauc_precision_at_100_diff1 - value: 12.347843899507293 - - type: nauc_precision_at_100_max - value: 27.717377508022622 - - type: nauc_precision_at_100_std - value: 26.73430752237529 - - type: nauc_precision_at_10_diff1 - value: 19.69177781807937 - - type: nauc_precision_at_10_max - value: 30.334030801241816 - - type: nauc_precision_at_10_std - value: 15.366237949875803 - - type: nauc_precision_at_1_diff1 - value: 33.63214806586087 - - type: nauc_precision_at_1_max - value: 28.809395842946238 - - type: nauc_precision_at_1_std - value: 3.870367319917128 - - type: nauc_precision_at_20_diff1 - value: 16.39420382179115 - - type: nauc_precision_at_20_max - value: 28.53821894442003 - - type: nauc_precision_at_20_std - value: 17.40002221290311 - - type: nauc_precision_at_3_diff1 - value: 25.01650633137198 - - type: nauc_precision_at_3_max - value: 27.40140981400113 - - type: nauc_precision_at_3_std - value: 9.528259668882319 - - type: nauc_precision_at_5_diff1 - value: 20.703008998648535 - - type: nauc_precision_at_5_max - value: 29.02824629392723 - - type: nauc_precision_at_5_std - value: 14.824369875578395 - - type: nauc_recall_at_1000_diff1 - value: 23.832697239355532 - - type: nauc_recall_at_1000_max - value: 44.08435745523949 - - type: nauc_recall_at_1000_std - value: 47.24836041537446 - - type: nauc_recall_at_100_diff1 - value: 16.300511580514367 - - type: nauc_recall_at_100_max - value: 35.91993875834976 - - type: nauc_recall_at_100_std - value: 31.711390146384005 - - type: nauc_recall_at_10_diff1 - value: 22.09945154154785 - - type: nauc_recall_at_10_max - value: 24.889396522698597 - - type: nauc_recall_at_10_std - value: 8.174508020746982 - - type: nauc_recall_at_1_diff1 - value: 37.47367952280333 - - type: nauc_recall_at_1_max - value: 28.526104067645214 - - type: nauc_recall_at_1_std - value: 0.42944400837957647 - - type: nauc_recall_at_20_diff1 - value: 20.215222984918444 - - type: nauc_recall_at_20_max - value: 24.29346815089969 - - type: nauc_recall_at_20_std - value: 9.6075370099546 - - type: nauc_recall_at_3_diff1 - value: 26.891648077306186 - - type: nauc_recall_at_3_max - value: 24.463956800351404 - - type: nauc_recall_at_3_std - value: 6.593385364536653 - - type: nauc_recall_at_5_diff1 - value: 22.088516789906684 - - type: nauc_recall_at_5_max - value: 24.856330338998195 - - type: nauc_recall_at_5_std - value: 8.935178303242722 - - type: ndcg_at_1 - value: 18.151 - - type: ndcg_at_10 - value: 27.023999999999997 - - type: ndcg_at_100 - value: 34.035 - - type: ndcg_at_1000 - value: 37.122 - - type: ndcg_at_20 - value: 29.688 - - type: ndcg_at_3 - value: 21.837 - - type: ndcg_at_5 - value: 23.995 - - type: precision_at_1 - value: 18.151 - - type: precision_at_10 - value: 5.479 - - type: precision_at_100 - value: 1.083 - - type: precision_at_1000 - value: 0.154 - - type: precision_at_20 - value: 3.5389999999999997 - - type: precision_at_3 - value: 10.693 - - type: precision_at_5 - value: 8.174 - - type: recall_at_1 - value: 14.198 - - type: recall_at_10 - value: 38.972 - - type: recall_at_100 - value: 68.97 - - type: recall_at_1000 - value: 90.291 - - type: recall_at_20 - value: 48.448 - - type: recall_at_3 - value: 24.421 - - type: recall_at_5 - value: 30.022 - task: - type: Retrieval - - dataset: - config: default - name: MTEB CQADupstackRetrieval (default) - revision: CQADupstackRetrieval_is_a_combined_dataset - split: test - type: CQADupstackRetrieval_is_a_combined_dataset - metrics: - - type: main_score - value: 26.12966666666667 - - type: ndcg_at_10 - value: 26.12966666666667 - task: - type: Retrieval - - dataset: - config: default - name: MTEB CQADupstackStatsRetrieval (default) - revision: 65ac3a16b8e91f9cee4c9828cc7c335575432a2a - split: test - type: mteb/cqadupstack-stats - metrics: - - type: main_score - value: 16.322 - - type: map_at_1 - value: 7.804 - - type: map_at_10 - value: 12.989 - - type: map_at_100 - value: 13.857 - - type: map_at_1000 - value: 14.001 - - type: map_at_20 - value: 13.388 - - type: map_at_3 - value: 11.192 - - type: map_at_5 - value: 12.176 - - type: mrr_at_1 - value: 9.049079754601227 - - type: mrr_at_10 - value: 14.444809621189995 - - type: mrr_at_100 - value: 15.361749681377326 - - type: mrr_at_1000 - value: 15.483726108258947 - - type: mrr_at_20 - value: 14.891287287777494 - - type: mrr_at_3 - value: 12.73006134969325 - - type: mrr_at_5 - value: 13.665644171779142 - - type: nauc_map_at_1000_diff1 - value: 22.283889437078248 - - type: nauc_map_at_1000_max - value: 14.944416577427388 - - type: nauc_map_at_1000_std - value: 7.317249110014133 - - type: nauc_map_at_100_diff1 - value: 22.28273180365645 - - type: nauc_map_at_100_max - value: 14.860297309097085 - - type: nauc_map_at_100_std - value: 7.3140105521353025 - - type: nauc_map_at_10_diff1 - value: 22.847655724529407 - - type: nauc_map_at_10_max - value: 15.378238244826022 - - type: nauc_map_at_10_std - value: 7.179195945065936 - - type: nauc_map_at_1_diff1 - value: 34.134583922704145 - - type: nauc_map_at_1_max - value: 17.044590989203783 - - type: nauc_map_at_1_std - value: 0.3675928894683656 - - type: nauc_map_at_20_diff1 - value: 22.355665146014907 - - type: nauc_map_at_20_max - value: 14.851743054913785 - - type: nauc_map_at_20_std - value: 7.270308478997049 - - type: nauc_map_at_3_diff1 - value: 24.73870536374159 - - type: nauc_map_at_3_max - value: 13.76122622256925 - - type: nauc_map_at_3_std - value: 4.850569295830115 - - type: nauc_map_at_5_diff1 - value: 23.46756043985898 - - type: nauc_map_at_5_max - value: 14.870499991558297 - - type: nauc_map_at_5_std - value: 5.31865354060947 - - type: nauc_mrr_at_1000_diff1 - value: 23.134711618650634 - - type: nauc_mrr_at_1000_max - value: 15.483448799540142 - - type: nauc_mrr_at_1000_std - value: 7.2969426798733705 - - type: nauc_mrr_at_100_diff1 - value: 23.11427541598592 - - type: nauc_mrr_at_100_max - value: 15.449183219736303 - - type: nauc_mrr_at_100_std - value: 7.319975335726775 - - type: nauc_mrr_at_10_diff1 - value: 23.488303603940714 - - type: nauc_mrr_at_10_max - value: 15.817711277401347 - - type: nauc_mrr_at_10_std - value: 6.933993376881063 - - type: nauc_mrr_at_1_diff1 - value: 33.31164870959659 - - type: nauc_mrr_at_1_max - value: 18.14101778006001 - - type: nauc_mrr_at_1_std - value: 3.5427712441050736 - - type: nauc_mrr_at_20_diff1 - value: 23.174358926183956 - - type: nauc_mrr_at_20_max - value: 15.311453081852944 - - type: nauc_mrr_at_20_std - value: 7.194856706727489 - - type: nauc_mrr_at_3_diff1 - value: 25.424783972258037 - - type: nauc_mrr_at_3_max - value: 15.54014171026033 - - type: nauc_mrr_at_3_std - value: 6.762746993243182 - - type: nauc_mrr_at_5_diff1 - value: 24.246811872930795 - - type: nauc_mrr_at_5_max - value: 15.62420277389364 - - type: nauc_mrr_at_5_std - value: 6.1118896243906 - - type: nauc_ndcg_at_1000_diff1 - value: 19.20841090878421 - - type: nauc_ndcg_at_1000_max - value: 16.374529426777627 - - type: nauc_ndcg_at_1000_std - value: 10.526588657800145 - - type: nauc_ndcg_at_100_diff1 - value: 18.922283216431406 - - type: nauc_ndcg_at_100_max - value: 14.097138336254844 - - type: nauc_ndcg_at_100_std - value: 10.389087631678793 - - type: nauc_ndcg_at_10_diff1 - value: 19.504289341336627 - - type: nauc_ndcg_at_10_max - value: 15.117288827566233 - - type: nauc_ndcg_at_10_std - value: 9.130368583693677 - - type: nauc_ndcg_at_1_diff1 - value: 33.31164870959659 - - type: nauc_ndcg_at_1_max - value: 18.14101778006001 - - type: nauc_ndcg_at_1_std - value: 3.5427712441050736 - - type: nauc_ndcg_at_20_diff1 - value: 18.233271885408556 - - type: nauc_ndcg_at_20_max - value: 13.419641851400083 - - type: nauc_ndcg_at_20_std - value: 9.624024533607203 - - type: nauc_ndcg_at_3_diff1 - value: 22.291734489026542 - - type: nauc_ndcg_at_3_max - value: 13.35741683411085 - - type: nauc_ndcg_at_3_std - value: 6.6206288867469665 - - type: nauc_ndcg_at_5_diff1 - value: 20.717344999706512 - - type: nauc_ndcg_at_5_max - value: 14.469746539869007 - - type: nauc_ndcg_at_5_std - value: 6.490356913357314 - - type: nauc_precision_at_1000_diff1 - value: 3.197341498400071 - - type: nauc_precision_at_1000_max - value: 24.13255446383312 - - type: nauc_precision_at_1000_std - value: 14.907711195306591 - - type: nauc_precision_at_100_diff1 - value: 10.56887700971008 - - type: nauc_precision_at_100_max - value: 18.003165201051015 - - type: nauc_precision_at_100_std - value: 22.32613638860972 - - type: nauc_precision_at_10_diff1 - value: 13.669635380022838 - - type: nauc_precision_at_10_max - value: 19.79091772206062 - - type: nauc_precision_at_10_std - value: 19.428145724937494 - - type: nauc_precision_at_1_diff1 - value: 33.31164870959659 - - type: nauc_precision_at_1_max - value: 18.14101778006001 - - type: nauc_precision_at_1_std - value: 3.5427712441050736 - - type: nauc_precision_at_20_diff1 - value: 10.171911848152861 - - type: nauc_precision_at_20_max - value: 15.434090781790694 - - type: nauc_precision_at_20_std - value: 20.113197023271713 - - type: nauc_precision_at_3_diff1 - value: 18.663239563832686 - - type: nauc_precision_at_3_max - value: 15.215643664801338 - - type: nauc_precision_at_3_std - value: 14.073704225451486 - - type: nauc_precision_at_5_diff1 - value: 15.741117074117424 - - type: nauc_precision_at_5_max - value: 18.083784453794294 - - type: nauc_precision_at_5_std - value: 14.170154968425852 - - type: nauc_recall_at_1000_diff1 - value: 12.529462519944262 - - type: nauc_recall_at_1000_max - value: 28.390833246662687 - - type: nauc_recall_at_1000_std - value: 22.515746615902728 - - type: nauc_recall_at_100_diff1 - value: 13.31762885233605 - - type: nauc_recall_at_100_max - value: 11.329638403083624 - - type: nauc_recall_at_100_std - value: 15.910452829837185 - - type: nauc_recall_at_10_diff1 - value: 11.902493318826277 - - type: nauc_recall_at_10_max - value: 13.620432998689278 - - type: nauc_recall_at_10_std - value: 12.254249902080712 - - type: nauc_recall_at_1_diff1 - value: 34.134583922704145 - - type: nauc_recall_at_1_max - value: 17.044590989203783 - - type: nauc_recall_at_1_std - value: 0.3675928894683656 - - type: nauc_recall_at_20_diff1 - value: 9.098713526218773 - - type: nauc_recall_at_20_max - value: 9.041415083058396 - - type: nauc_recall_at_20_std - value: 13.431500488643977 - - type: nauc_recall_at_3_diff1 - value: 16.13589077711093 - - type: nauc_recall_at_3_max - value: 9.03348176243661 - - type: nauc_recall_at_3_std - value: 7.3960632043526155 - - type: nauc_recall_at_5_diff1 - value: 14.140180575248207 - - type: nauc_recall_at_5_max - value: 11.845879266846513 - - type: nauc_recall_at_5_std - value: 7.0735189487104 - - type: ndcg_at_1 - value: 9.049 - - type: ndcg_at_10 - value: 16.322 - - type: ndcg_at_100 - value: 21.312 - - type: ndcg_at_1000 - value: 25.323 - - type: ndcg_at_20 - value: 17.763 - - type: ndcg_at_3 - value: 12.792 - - type: ndcg_at_5 - value: 14.45 - - type: precision_at_1 - value: 9.049 - - type: precision_at_10 - value: 3.083 - - type: precision_at_100 - value: 0.623 - - type: precision_at_1000 - value: 0.106 - - type: precision_at_20 - value: 1.902 - - type: precision_at_3 - value: 6.237 - - type: precision_at_5 - value: 4.784999999999999 - - type: recall_at_1 - value: 7.804 - - type: recall_at_10 - value: 25.28 - - type: recall_at_100 - value: 49.108000000000004 - - type: recall_at_1000 - value: 79.468 - - type: recall_at_20 - value: 30.605 - - type: recall_at_3 - value: 15.626999999999999 - - type: recall_at_5 - value: 19.529 - task: - type: Retrieval - - dataset: - config: default - name: MTEB CQADupstackTexRetrieval (default) - revision: 46989137a86843e03a6195de44b09deda022eec7 - split: test - type: mteb/cqadupstack-tex - metrics: - - type: main_score - value: 13.761999999999999 - - type: map_at_1 - value: 6.3759999999999994 - - type: map_at_10 - value: 10.558 - - type: map_at_100 - value: 11.532 - - type: map_at_1000 - value: 11.695 - - type: map_at_20 - value: 11.028 - - type: map_at_3 - value: 8.879 - - type: map_at_5 - value: 9.713 - - type: mrr_at_1 - value: 8.224363386097728 - - type: mrr_at_10 - value: 13.012431859644524 - - type: mrr_at_100 - value: 13.946814841802762 - - type: mrr_at_1000 - value: 14.06290442969105 - - type: mrr_at_20 - value: 13.497735030482463 - - type: mrr_at_3 - value: 11.155081440697408 - - type: mrr_at_5 - value: 12.115164028446872 - - type: nauc_map_at_1000_diff1 - value: 23.43507587077875 - - type: nauc_map_at_1000_max - value: 20.43870564782279 - - type: nauc_map_at_1000_std - value: 12.873082418993908 - - type: nauc_map_at_100_diff1 - value: 23.438050985565148 - - type: nauc_map_at_100_max - value: 20.399313617924037 - - type: nauc_map_at_100_std - value: 12.722487279959077 - - type: nauc_map_at_10_diff1 - value: 24.205691744325318 - - type: nauc_map_at_10_max - value: 20.399611787740103 - - type: nauc_map_at_10_std - value: 12.408592197191174 - - type: nauc_map_at_1_diff1 - value: 34.071984947286296 - - type: nauc_map_at_1_max - value: 22.35927499649868 - - type: nauc_map_at_1_std - value: 9.086052394259115 - - type: nauc_map_at_20_diff1 - value: 23.5442558068055 - - type: nauc_map_at_20_max - value: 20.248891696767902 - - type: nauc_map_at_20_std - value: 12.51101167730077 - - type: nauc_map_at_3_diff1 - value: 26.482256969753188 - - type: nauc_map_at_3_max - value: 21.208222413456188 - - type: nauc_map_at_3_std - value: 10.787698162827862 - - type: nauc_map_at_5_diff1 - value: 25.11166942901193 - - type: nauc_map_at_5_max - value: 20.84541422714762 - - type: nauc_map_at_5_std - value: 11.084044604237206 - - type: nauc_mrr_at_1000_diff1 - value: 21.650853612058675 - - type: nauc_mrr_at_1000_max - value: 21.417971870794876 - - type: nauc_mrr_at_1000_std - value: 15.501648201831497 - - type: nauc_mrr_at_100_diff1 - value: 21.61532662748058 - - type: nauc_mrr_at_100_max - value: 21.407856995308325 - - type: nauc_mrr_at_100_std - value: 15.479695245125175 - - type: nauc_mrr_at_10_diff1 - value: 22.115984471726797 - - type: nauc_mrr_at_10_max - value: 21.404557414782232 - - type: nauc_mrr_at_10_std - value: 15.479987126659434 - - type: nauc_mrr_at_1_diff1 - value: 30.402870566797606 - - type: nauc_mrr_at_1_max - value: 23.93938440196226 - - type: nauc_mrr_at_1_std - value: 12.997536237455929 - - type: nauc_mrr_at_20_diff1 - value: 21.601037898265016 - - type: nauc_mrr_at_20_max - value: 21.376546238813333 - - type: nauc_mrr_at_20_std - value: 15.519106032577042 - - type: nauc_mrr_at_3_diff1 - value: 24.391327825821342 - - type: nauc_mrr_at_3_max - value: 22.279324585023907 - - type: nauc_mrr_at_3_std - value: 14.36723796962747 - - type: nauc_mrr_at_5_diff1 - value: 22.69779016609937 - - type: nauc_mrr_at_5_max - value: 21.910155676991927 - - type: nauc_mrr_at_5_std - value: 14.481452222550335 - - type: nauc_ndcg_at_1000_diff1 - value: 18.646473335235644 - - type: nauc_ndcg_at_1000_max - value: 20.61450878843323 - - type: nauc_ndcg_at_1000_std - value: 17.02503878660379 - - type: nauc_ndcg_at_100_diff1 - value: 18.516435075748475 - - type: nauc_ndcg_at_100_max - value: 19.916591730147978 - - type: nauc_ndcg_at_100_std - value: 15.365433816151405 - - type: nauc_ndcg_at_10_diff1 - value: 20.433623492398315 - - type: nauc_ndcg_at_10_max - value: 19.505014482656055 - - type: nauc_ndcg_at_10_std - value: 15.007091547236504 - - type: nauc_ndcg_at_1_diff1 - value: 30.402870566797606 - - type: nauc_ndcg_at_1_max - value: 23.93938440196226 - - type: nauc_ndcg_at_1_std - value: 12.997536237455929 - - type: nauc_ndcg_at_20_diff1 - value: 18.786357992932725 - - type: nauc_ndcg_at_20_max - value: 19.098773770805913 - - type: nauc_ndcg_at_20_std - value: 15.077078565809456 - - type: nauc_ndcg_at_3_diff1 - value: 23.72149188171607 - - type: nauc_ndcg_at_3_max - value: 21.3883319853042 - - type: nauc_ndcg_at_3_std - value: 12.898024573912107 - - type: nauc_ndcg_at_5_diff1 - value: 21.68066258367858 - - type: nauc_ndcg_at_5_max - value: 20.613614404831925 - - type: nauc_ndcg_at_5_std - value: 12.774272028171833 - - type: nauc_precision_at_1000_diff1 - value: 0.9792326314146174 - - type: nauc_precision_at_1000_max - value: 18.959462451645233 - - type: nauc_precision_at_1000_std - value: 27.112339675868473 - - type: nauc_precision_at_100_diff1 - value: 5.507315634114162 - - type: nauc_precision_at_100_max - value: 21.16546753595108 - - type: nauc_precision_at_100_std - value: 23.890261020569653 - - type: nauc_precision_at_10_diff1 - value: 12.416990446480753 - - type: nauc_precision_at_10_max - value: 20.972089855286185 - - type: nauc_precision_at_10_std - value: 23.051845398601355 - - type: nauc_precision_at_1_diff1 - value: 30.402870566797606 - - type: nauc_precision_at_1_max - value: 23.93938440196226 - - type: nauc_precision_at_1_std - value: 12.997536237455929 - - type: nauc_precision_at_20_diff1 - value: 8.79422472044008 - - type: nauc_precision_at_20_max - value: 20.674903676907636 - - type: nauc_precision_at_20_std - value: 23.488209998563608 - - type: nauc_precision_at_3_diff1 - value: 18.02293696627094 - - type: nauc_precision_at_3_max - value: 22.950256317548767 - - type: nauc_precision_at_3_std - value: 17.001251788226636 - - type: nauc_precision_at_5_diff1 - value: 14.82261165916923 - - type: nauc_precision_at_5_max - value: 22.371225776931077 - - type: nauc_precision_at_5_std - value: 17.806150673765604 - - type: nauc_recall_at_1000_diff1 - value: 7.494489915270187 - - type: nauc_recall_at_1000_max - value: 20.569342570845002 - - type: nauc_recall_at_1000_std - value: 25.866094507869892 - - type: nauc_recall_at_100_diff1 - value: 10.14085619919932 - - type: nauc_recall_at_100_max - value: 16.931051365465137 - - type: nauc_recall_at_100_std - value: 15.888964104887746 - - type: nauc_recall_at_10_diff1 - value: 14.225307287972269 - - type: nauc_recall_at_10_max - value: 15.52986134762012 - - type: nauc_recall_at_10_std - value: 16.71753091718452 - - type: nauc_recall_at_1_diff1 - value: 34.071984947286296 - - type: nauc_recall_at_1_max - value: 22.35927499649868 - - type: nauc_recall_at_1_std - value: 9.086052394259115 - - type: nauc_recall_at_20_diff1 - value: 10.539561215408414 - - type: nauc_recall_at_20_max - value: 14.542633350220521 - - type: nauc_recall_at_20_std - value: 16.455084074160734 - - type: nauc_recall_at_3_diff1 - value: 19.865442369243137 - - type: nauc_recall_at_3_max - value: 18.637208048866047 - - type: nauc_recall_at_3_std - value: 11.885469722659007 - - type: nauc_recall_at_5_diff1 - value: 16.637796934858315 - - type: nauc_recall_at_5_max - value: 17.833140476248197 - - type: nauc_recall_at_5_std - value: 12.219010395478854 - - type: ndcg_at_1 - value: 8.224 - - type: ndcg_at_10 - value: 13.761999999999999 - - type: ndcg_at_100 - value: 19.006 - - type: ndcg_at_1000 - value: 23.144000000000002 - - type: ndcg_at_20 - value: 15.414 - - type: ndcg_at_3 - value: 10.464 - - type: ndcg_at_5 - value: 11.798 - - type: precision_at_1 - value: 8.224 - - type: precision_at_10 - value: 2.9899999999999998 - - type: precision_at_100 - value: 0.6890000000000001 - - type: precision_at_1000 - value: 0.124 - - type: precision_at_20 - value: 1.958 - - type: precision_at_3 - value: 5.367999999999999 - - type: precision_at_5 - value: 4.2459999999999996 - - type: recall_at_1 - value: 6.3759999999999994 - - type: recall_at_10 - value: 21.159 - - type: recall_at_100 - value: 45.506 - - type: recall_at_1000 - value: 75.702 - - type: recall_at_20 - value: 27.349 - - type: recall_at_3 - value: 11.837 - - type: recall_at_5 - value: 15.232000000000001 - task: - type: Retrieval - - dataset: - config: default - name: MTEB CQADupstackUnixRetrieval (default) - revision: 6c6430d3a6d36f8d2a829195bc5dc94d7e063e53 - split: test - type: mteb/cqadupstack-unix - metrics: - - type: main_score - value: 27.52 - - type: map_at_1 - value: 15.588 - - type: map_at_10 - value: 22.570999999999998 - - type: map_at_100 - value: 23.982999999999997 - - type: map_at_1000 - value: 24.122 - - type: map_at_20 - value: 23.391000000000002 - - type: map_at_3 - value: 19.767000000000003 - - type: map_at_5 - value: 21.18 - - type: mrr_at_1 - value: 18.75 - - type: mrr_at_10 - value: 26.1608623548922 - - type: mrr_at_100 - value: 27.317010497161142 - - type: mrr_at_1000 - value: 27.405975383784188 - - type: mrr_at_20 - value: 26.87336000217942 - - type: mrr_at_3 - value: 23.445273631840813 - - type: mrr_at_5 - value: 24.8212064676617 - - type: nauc_map_at_1000_diff1 - value: 30.684226668507026 - - type: nauc_map_at_1000_max - value: 24.951065677373784 - - type: nauc_map_at_1000_std - value: 5.2066026685896825 - - type: nauc_map_at_100_diff1 - value: 30.679165862866476 - - type: nauc_map_at_100_max - value: 24.9530364766238 - - type: nauc_map_at_100_std - value: 5.205883565949459 - - type: nauc_map_at_10_diff1 - value: 31.122215360656153 - - type: nauc_map_at_10_max - value: 24.5124066227212 - - type: nauc_map_at_10_std - value: 4.35292624287149 - - type: nauc_map_at_1_diff1 - value: 39.18247834731389 - - type: nauc_map_at_1_max - value: 24.413907066366434 - - type: nauc_map_at_1_std - value: 2.070371372630658 - - type: nauc_map_at_20_diff1 - value: 30.818586802122503 - - type: nauc_map_at_20_max - value: 24.77393316529099 - - type: nauc_map_at_20_std - value: 4.935672359462836 - - type: nauc_map_at_3_diff1 - value: 31.99317691078875 - - type: nauc_map_at_3_max - value: 23.376773341655184 - - type: nauc_map_at_3_std - value: 1.7026914448918886 - - type: nauc_map_at_5_diff1 - value: 31.508091004529398 - - type: nauc_map_at_5_max - value: 24.560067495036396 - - type: nauc_map_at_5_std - value: 3.552337846934037 - - type: nauc_mrr_at_1000_diff1 - value: 28.985987949736963 - - type: nauc_mrr_at_1000_max - value: 27.028330536592133 - - type: nauc_mrr_at_1000_std - value: 6.805769019477874 - - type: nauc_mrr_at_100_diff1 - value: 28.975617833163696 - - type: nauc_mrr_at_100_max - value: 27.03219548034752 - - type: nauc_mrr_at_100_std - value: 6.831150418834171 - - type: nauc_mrr_at_10_diff1 - value: 29.030864214165074 - - type: nauc_mrr_at_10_max - value: 26.77447636537233 - - type: nauc_mrr_at_10_std - value: 6.407525890746923 - - type: nauc_mrr_at_1_diff1 - value: 36.43774473918464 - - type: nauc_mrr_at_1_max - value: 27.70874373843583 - - type: nauc_mrr_at_1_std - value: 4.839051843867864 - - type: nauc_mrr_at_20_diff1 - value: 29.016976637513753 - - type: nauc_mrr_at_20_max - value: 27.004877168874504 - - type: nauc_mrr_at_20_std - value: 6.775422720382143 - - type: nauc_mrr_at_3_diff1 - value: 30.077493911870413 - - type: nauc_mrr_at_3_max - value: 26.340420675504696 - - type: nauc_mrr_at_3_std - value: 4.141758328637211 - - type: nauc_mrr_at_5_diff1 - value: 29.434648381968586 - - type: nauc_mrr_at_5_max - value: 27.04904363072304 - - type: nauc_mrr_at_5_std - value: 5.638876519130593 - - type: nauc_ndcg_at_1000_diff1 - value: 27.201512870428225 - - type: nauc_ndcg_at_1000_max - value: 26.607429208642486 - - type: nauc_ndcg_at_1000_std - value: 9.426530569631955 - - type: nauc_ndcg_at_100_diff1 - value: 26.7561672878664 - - type: nauc_ndcg_at_100_max - value: 26.82765674698552 - - type: nauc_ndcg_at_100_std - value: 10.117411449672364 - - type: nauc_ndcg_at_10_diff1 - value: 28.08039219869409 - - type: nauc_ndcg_at_10_max - value: 25.30612250538394 - - type: nauc_ndcg_at_10_std - value: 6.8541921175323655 - - type: nauc_ndcg_at_1_diff1 - value: 36.43774473918464 - - type: nauc_ndcg_at_1_max - value: 27.70874373843583 - - type: nauc_ndcg_at_1_std - value: 4.839051843867864 - - type: nauc_ndcg_at_20_diff1 - value: 27.27399749691096 - - type: nauc_ndcg_at_20_max - value: 26.107141341582334 - - type: nauc_ndcg_at_20_std - value: 8.649506543910311 - - type: nauc_ndcg_at_3_diff1 - value: 29.424613119884025 - - type: nauc_ndcg_at_3_max - value: 24.17384798641367 - - type: nauc_ndcg_at_3_std - value: 2.3404905076142164 - - type: nauc_ndcg_at_5_diff1 - value: 28.77369502430592 - - type: nauc_ndcg_at_5_max - value: 25.64790019257201 - - type: nauc_ndcg_at_5_std - value: 5.155072577708316 - - type: nauc_precision_at_1000_diff1 - value: -4.7351598070789285 - - type: nauc_precision_at_1000_max - value: 3.8148394654088924 - - type: nauc_precision_at_1000_std - value: 5.833941375739573 - - type: nauc_precision_at_100_diff1 - value: 2.7442597276716962 - - type: nauc_precision_at_100_max - value: 21.287933846054695 - - type: nauc_precision_at_100_std - value: 20.177490390686852 - - type: nauc_precision_at_10_diff1 - value: 15.735811418920292 - - type: nauc_precision_at_10_max - value: 27.22995218775853 - - type: nauc_precision_at_10_std - value: 14.258744772316767 - - type: nauc_precision_at_1_diff1 - value: 36.43774473918464 - - type: nauc_precision_at_1_max - value: 27.70874373843583 - - type: nauc_precision_at_1_std - value: 4.839051843867864 - - type: nauc_precision_at_20_diff1 - value: 11.68099186992869 - - type: nauc_precision_at_20_max - value: 27.11541433113338 - - type: nauc_precision_at_20_std - value: 18.26747110990696 - - type: nauc_precision_at_3_diff1 - value: 22.10511897461343 - - type: nauc_precision_at_3_max - value: 26.17867919555923 - - type: nauc_precision_at_3_std - value: 4.707255449070724 - - type: nauc_precision_at_5_diff1 - value: 19.791048476417362 - - type: nauc_precision_at_5_max - value: 29.234327716637033 - - type: nauc_precision_at_5_std - value: 10.540070304399105 - - type: nauc_recall_at_1000_diff1 - value: 8.421470865964084 - - type: nauc_recall_at_1000_max - value: 35.84068473670827 - - type: nauc_recall_at_1000_std - value: 43.8492485408352 - - type: nauc_recall_at_100_diff1 - value: 14.057683903084895 - - type: nauc_recall_at_100_max - value: 29.18102905270184 - - type: nauc_recall_at_100_std - value: 26.137740725020237 - - type: nauc_recall_at_10_diff1 - value: 20.883794816820973 - - type: nauc_recall_at_10_max - value: 23.209326744403462 - - type: nauc_recall_at_10_std - value: 10.744448122178655 - - type: nauc_recall_at_1_diff1 - value: 39.18247834731389 - - type: nauc_recall_at_1_max - value: 24.413907066366434 - - type: nauc_recall_at_1_std - value: 2.070371372630658 - - type: nauc_recall_at_20_diff1 - value: 17.615849661206664 - - type: nauc_recall_at_20_max - value: 25.238448658847183 - - type: nauc_recall_at_20_std - value: 16.41969229374057 - - type: nauc_recall_at_3_diff1 - value: 24.620971173135203 - - type: nauc_recall_at_3_max - value: 21.570609052956133 - - type: nauc_recall_at_3_std - value: 1.1341134767333612 - - type: nauc_recall_at_5_diff1 - value: 23.02632411597575 - - type: nauc_recall_at_5_max - value: 25.011330440228207 - - type: nauc_recall_at_5_std - value: 7.4900296759785725 - - type: ndcg_at_1 - value: 18.75 - - type: ndcg_at_10 - value: 27.52 - - type: ndcg_at_100 - value: 34.076 - - type: ndcg_at_1000 - value: 37.259 - - type: ndcg_at_20 - value: 30.266 - - type: ndcg_at_3 - value: 22.186 - - type: ndcg_at_5 - value: 24.39 - - type: precision_at_1 - value: 18.75 - - type: precision_at_10 - value: 5.177 - - type: precision_at_100 - value: 0.9530000000000001 - - type: precision_at_1000 - value: 0.136 - - type: precision_at_20 - value: 3.279 - - type: precision_at_3 - value: 10.354 - - type: precision_at_5 - value: 7.743 - - type: recall_at_1 - value: 15.588 - - type: recall_at_10 - value: 39.202 - - type: recall_at_100 - value: 68.057 - - type: recall_at_1000 - value: 90.32900000000001 - - type: recall_at_20 - value: 49.331 - - type: recall_at_3 - value: 24.543 - - type: recall_at_5 - value: 30.133 - task: - type: Retrieval - - dataset: - config: default - name: MTEB CQADupstackWebmastersRetrieval (default) - revision: 160c094312a0e1facb97e55eeddb698c0abe3571 - split: test - type: mteb/cqadupstack-webmasters - metrics: - - type: main_score - value: 27.418 - - type: map_at_1 - value: 15.928 - - type: map_at_10 - value: 22.445 - - type: map_at_100 - value: 23.729 - - type: map_at_1000 - value: 23.945 - - type: map_at_20 - value: 23.097 - - type: map_at_3 - value: 19.742 - - type: map_at_5 - value: 21.187 - - type: mrr_at_1 - value: 19.76284584980237 - - type: mrr_at_10 - value: 26.411945542380323 - - type: mrr_at_100 - value: 27.542176582406203 - - type: mrr_at_1000 - value: 27.621234648152367 - - type: mrr_at_20 - value: 27.047537563344783 - - type: mrr_at_3 - value: 23.781291172595513 - - type: mrr_at_5 - value: 25.243741765480888 - - type: nauc_map_at_1000_diff1 - value: 34.875954784511165 - - type: nauc_map_at_1000_max - value: 29.402608286733056 - - type: nauc_map_at_1000_std - value: 18.888924007482892 - - type: nauc_map_at_100_diff1 - value: 34.67273650132461 - - type: nauc_map_at_100_max - value: 29.36883910278622 - - type: nauc_map_at_100_std - value: 18.801793750158218 - - type: nauc_map_at_10_diff1 - value: 34.925919015533104 - - type: nauc_map_at_10_max - value: 28.906142009894193 - - type: nauc_map_at_10_std - value: 18.47240468202337 - - type: nauc_map_at_1_diff1 - value: 41.320257922377046 - - type: nauc_map_at_1_max - value: 27.045525641366126 - - type: nauc_map_at_1_std - value: 14.820719523595283 - - type: nauc_map_at_20_diff1 - value: 34.38606553153665 - - type: nauc_map_at_20_max - value: 29.339130166316725 - - type: nauc_map_at_20_std - value: 18.777126514716354 - - type: nauc_map_at_3_diff1 - value: 36.433641134537105 - - type: nauc_map_at_3_max - value: 28.429893730930715 - - type: nauc_map_at_3_std - value: 16.573807982457755 - - type: nauc_map_at_5_diff1 - value: 35.49829863276235 - - type: nauc_map_at_5_max - value: 28.49206642708423 - - type: nauc_map_at_5_std - value: 17.426074185877614 - - type: nauc_mrr_at_1000_diff1 - value: 33.68862596978553 - - type: nauc_mrr_at_1000_max - value: 32.25195024978712 - - type: nauc_mrr_at_1000_std - value: 19.920510902611937 - - type: nauc_mrr_at_100_diff1 - value: 33.69603987988904 - - type: nauc_mrr_at_100_max - value: 32.26963439157751 - - type: nauc_mrr_at_100_std - value: 19.963818748298817 - - type: nauc_mrr_at_10_diff1 - value: 33.96876056002211 - - type: nauc_mrr_at_10_max - value: 32.103845582717376 - - type: nauc_mrr_at_10_std - value: 19.742125901752846 - - type: nauc_mrr_at_1_diff1 - value: 37.82438868347941 - - type: nauc_mrr_at_1_max - value: 33.02998941581047 - - type: nauc_mrr_at_1_std - value: 16.995528862845454 - - type: nauc_mrr_at_20_diff1 - value: 33.596380686430635 - - type: nauc_mrr_at_20_max - value: 32.30918088421744 - - type: nauc_mrr_at_20_std - value: 19.97601283298697 - - type: nauc_mrr_at_3_diff1 - value: 34.6722198156948 - - type: nauc_mrr_at_3_max - value: 31.94335032060688 - - type: nauc_mrr_at_3_std - value: 17.957820211447814 - - type: nauc_mrr_at_5_diff1 - value: 34.16743584432419 - - type: nauc_mrr_at_5_max - value: 32.025706005037726 - - type: nauc_mrr_at_5_std - value: 19.012154164577908 - - type: nauc_ndcg_at_1000_diff1 - value: 32.826999318624274 - - type: nauc_ndcg_at_1000_max - value: 30.765793533727216 - - type: nauc_ndcg_at_1000_std - value: 22.04794628482662 - - type: nauc_ndcg_at_100_diff1 - value: 32.475577778913475 - - type: nauc_ndcg_at_100_max - value: 30.914747692829447 - - type: nauc_ndcg_at_100_std - value: 22.335685545222717 - - type: nauc_ndcg_at_10_diff1 - value: 32.68201159086846 - - type: nauc_ndcg_at_10_max - value: 29.58944485699082 - - type: nauc_ndcg_at_10_std - value: 21.446283835382552 - - type: nauc_ndcg_at_1_diff1 - value: 37.82438868347941 - - type: nauc_ndcg_at_1_max - value: 33.02998941581047 - - type: nauc_ndcg_at_1_std - value: 16.995528862845454 - - type: nauc_ndcg_at_20_diff1 - value: 30.923091310022038 - - type: nauc_ndcg_at_20_max - value: 30.583718384467428 - - type: nauc_ndcg_at_20_std - value: 22.307175012464036 - - type: nauc_ndcg_at_3_diff1 - value: 34.76421528647677 - - type: nauc_ndcg_at_3_max - value: 30.087603886151175 - - type: nauc_ndcg_at_3_std - value: 18.7195367899851 - - type: nauc_ndcg_at_5_diff1 - value: 33.73582113410747 - - type: nauc_ndcg_at_5_max - value: 29.396452917503417 - - type: nauc_ndcg_at_5_std - value: 19.62391988291154 - - type: nauc_precision_at_1000_diff1 - value: 32.33220510193037 - - type: nauc_precision_at_1000_max - value: 9.623028966785611 - - type: nauc_precision_at_1000_std - value: 10.741264545736154 - - type: nauc_precision_at_100_diff1 - value: 23.98156772823854 - - type: nauc_precision_at_100_max - value: 22.227294958441433 - - type: nauc_precision_at_100_std - value: 20.635069438588186 - - type: nauc_precision_at_10_diff1 - value: 22.347016279919913 - - type: nauc_precision_at_10_max - value: 31.586051361410473 - - type: nauc_precision_at_10_std - value: 26.869528206483544 - - type: nauc_precision_at_1_diff1 - value: 37.82438868347941 - - type: nauc_precision_at_1_max - value: 33.02998941581047 - - type: nauc_precision_at_1_std - value: 16.995528862845454 - - type: nauc_precision_at_20_diff1 - value: 18.419666290305088 - - type: nauc_precision_at_20_max - value: 32.321771618176115 - - type: nauc_precision_at_20_std - value: 29.47795655526015 - - type: nauc_precision_at_3_diff1 - value: 27.863265709133366 - - type: nauc_precision_at_3_max - value: 35.285838551821435 - - type: nauc_precision_at_3_std - value: 22.672769836029126 - - type: nauc_precision_at_5_diff1 - value: 25.640021400595604 - - type: nauc_precision_at_5_max - value: 33.24005429566627 - - type: nauc_precision_at_5_std - value: 23.121187346809773 - - type: nauc_recall_at_1000_diff1 - value: 12.233272868637059 - - type: nauc_recall_at_1000_max - value: 31.34004050188055 - - type: nauc_recall_at_1000_std - value: 38.94602574688917 - - type: nauc_recall_at_100_diff1 - value: 23.47720048321073 - - type: nauc_recall_at_100_max - value: 30.71446489259641 - - type: nauc_recall_at_100_std - value: 31.928015150518757 - - type: nauc_recall_at_10_diff1 - value: 24.73037700905113 - - type: nauc_recall_at_10_max - value: 26.466497802514382 - - type: nauc_recall_at_10_std - value: 25.65417986790167 - - type: nauc_recall_at_1_diff1 - value: 41.320257922377046 - - type: nauc_recall_at_1_max - value: 27.045525641366126 - - type: nauc_recall_at_1_std - value: 14.820719523595283 - - type: nauc_recall_at_20_diff1 - value: 17.78335919785901 - - type: nauc_recall_at_20_max - value: 28.91031250817357 - - type: nauc_recall_at_20_std - value: 28.823907515459098 - - type: nauc_recall_at_3_diff1 - value: 31.412269010866538 - - type: nauc_recall_at_3_max - value: 26.226141310560102 - - type: nauc_recall_at_3_std - value: 17.606845122657564 - - type: nauc_recall_at_5_diff1 - value: 27.631512319754588 - - type: nauc_recall_at_5_max - value: 25.456300010644167 - - type: nauc_recall_at_5_std - value: 20.554194945871142 - - type: ndcg_at_1 - value: 19.763 - - type: ndcg_at_10 - value: 27.418 - - type: ndcg_at_100 - value: 33.794999999999995 - - type: ndcg_at_1000 - value: 37.206 - - type: ndcg_at_20 - value: 29.497 - - type: ndcg_at_3 - value: 22.594 - - type: ndcg_at_5 - value: 24.782 - - type: precision_at_1 - value: 19.763 - - type: precision_at_10 - value: 5.415 - - type: precision_at_100 - value: 1.154 - - type: precision_at_1000 - value: 0.22 - - type: precision_at_20 - value: 3.35 - - type: precision_at_3 - value: 10.804 - - type: precision_at_5 - value: 8.221 - - type: recall_at_1 - value: 15.928 - - type: recall_at_10 - value: 37.972 - - type: recall_at_100 - value: 68.31700000000001 - - type: recall_at_1000 - value: 89.744 - - type: recall_at_20 - value: 46.399 - - type: recall_at_3 - value: 23.796 - - type: recall_at_5 - value: 29.528 - task: - type: Retrieval - - dataset: - config: default - name: MTEB CQADupstackWordpressRetrieval (default) - revision: 4ffe81d471b1924886b33c7567bfb200e9eec5c4 - split: test - type: mteb/cqadupstack-wordpress - metrics: - - type: main_score - value: 21.26 - - type: map_at_1 - value: 10.974 - - type: map_at_10 - value: 17.136000000000003 - - type: map_at_100 - value: 18.177 - - type: map_at_1000 - value: 18.287 - - type: map_at_20 - value: 17.636 - - type: map_at_3 - value: 14.934 - - type: map_at_5 - value: 16.073 - - type: mrr_at_1 - value: 12.199630314232902 - - type: mrr_at_10 - value: 18.70514626059913 - - type: mrr_at_100 - value: 19.748983258794034 - - type: mrr_at_1000 - value: 19.837442070088112 - - type: mrr_at_20 - value: 19.248603491362864 - - type: mrr_at_3 - value: 16.54343807763401 - - type: mrr_at_5 - value: 17.670979667282815 - - type: nauc_map_at_1000_diff1 - value: 12.678931653901056 - - type: nauc_map_at_1000_max - value: 16.188177166378136 - - type: nauc_map_at_1000_std - value: 12.840915498598127 - - type: nauc_map_at_100_diff1 - value: 12.670776495345507 - - type: nauc_map_at_100_max - value: 16.17798043496996 - - type: nauc_map_at_100_std - value: 12.822072072092977 - - type: nauc_map_at_10_diff1 - value: 12.288240379882705 - - type: nauc_map_at_10_max - value: 16.15504212741907 - - type: nauc_map_at_10_std - value: 12.164525297458955 - - type: nauc_map_at_1_diff1 - value: 19.07356377322629 - - type: nauc_map_at_1_max - value: 17.999572753995082 - - type: nauc_map_at_1_std - value: 10.302044048546005 - - type: nauc_map_at_20_diff1 - value: 12.310741859813563 - - type: nauc_map_at_20_max - value: 15.874183751438679 - - type: nauc_map_at_20_std - value: 12.329892661885577 - - type: nauc_map_at_3_diff1 - value: 12.923055965922023 - - type: nauc_map_at_3_max - value: 16.030670921089712 - - type: nauc_map_at_3_std - value: 10.174236364706072 - - type: nauc_map_at_5_diff1 - value: 12.334399032409554 - - type: nauc_map_at_5_max - value: 15.847592809708466 - - type: nauc_map_at_5_std - value: 10.642632496675084 - - type: nauc_mrr_at_1000_diff1 - value: 14.821689314895858 - - type: nauc_mrr_at_1000_max - value: 18.01007338288078 - - type: nauc_mrr_at_1000_std - value: 12.596964429973683 - - type: nauc_mrr_at_100_diff1 - value: 14.813075628659917 - - type: nauc_mrr_at_100_max - value: 17.987900378330597 - - type: nauc_mrr_at_100_std - value: 12.585770827205087 - - type: nauc_mrr_at_10_diff1 - value: 14.407724617483183 - - type: nauc_mrr_at_10_max - value: 17.9987939329453 - - type: nauc_mrr_at_10_std - value: 12.295346246957186 - - type: nauc_mrr_at_1_diff1 - value: 21.868324878950887 - - type: nauc_mrr_at_1_max - value: 20.894193908164336 - - type: nauc_mrr_at_1_std - value: 9.139550923680906 - - type: nauc_mrr_at_20_diff1 - value: 14.570968072256132 - - type: nauc_mrr_at_20_max - value: 17.744023038753383 - - type: nauc_mrr_at_20_std - value: 12.120612537829208 - - type: nauc_mrr_at_3_diff1 - value: 15.73722761425632 - - type: nauc_mrr_at_3_max - value: 18.197994422315624 - - type: nauc_mrr_at_3_std - value: 10.196475686377902 - - type: nauc_mrr_at_5_diff1 - value: 14.745400930877594 - - type: nauc_mrr_at_5_max - value: 17.907279586527572 - - type: nauc_mrr_at_5_std - value: 11.13820235180636 - - type: nauc_ndcg_at_1000_diff1 - value: 12.13214616065359 - - type: nauc_ndcg_at_1000_max - value: 17.121764777921207 - - type: nauc_ndcg_at_1000_std - value: 17.893050819270822 - - type: nauc_ndcg_at_100_diff1 - value: 11.950146646556552 - - type: nauc_ndcg_at_100_max - value: 16.509372959176655 - - type: nauc_ndcg_at_100_std - value: 17.882671216027205 - - type: nauc_ndcg_at_10_diff1 - value: 10.280723820236313 - - type: nauc_ndcg_at_10_max - value: 16.19992373296763 - - type: nauc_ndcg_at_10_std - value: 14.772336098381986 - - type: nauc_ndcg_at_1_diff1 - value: 21.868324878950887 - - type: nauc_ndcg_at_1_max - value: 20.894193908164336 - - type: nauc_ndcg_at_1_std - value: 9.139550923680906 - - type: nauc_ndcg_at_20_diff1 - value: 10.4067889502451 - - type: nauc_ndcg_at_20_max - value: 15.234819276730866 - - type: nauc_ndcg_at_20_std - value: 14.906728449046724 - - type: nauc_ndcg_at_3_diff1 - value: 11.811602705354899 - - type: nauc_ndcg_at_3_max - value: 16.046989278985375 - - type: nauc_ndcg_at_3_std - value: 10.67288489841697 - - type: nauc_ndcg_at_5_diff1 - value: 10.513005314547621 - - type: nauc_ndcg_at_5_max - value: 15.506542272361076 - - type: nauc_ndcg_at_5_std - value: 11.51643304336521 - - type: nauc_precision_at_1000_diff1 - value: 6.230871653536136 - - type: nauc_precision_at_1000_max - value: 2.8786312954363806 - - type: nauc_precision_at_1000_std - value: 8.072311757002275 - - type: nauc_precision_at_100_diff1 - value: 12.69414692799929 - - type: nauc_precision_at_100_max - value: 15.001693281013653 - - type: nauc_precision_at_100_std - value: 26.434565117841185 - - type: nauc_precision_at_10_diff1 - value: 7.8769498354922485 - - type: nauc_precision_at_10_max - value: 16.51010130232648 - - type: nauc_precision_at_10_std - value: 20.951244649430134 - - type: nauc_precision_at_1_diff1 - value: 21.868324878950887 - - type: nauc_precision_at_1_max - value: 20.894193908164336 - - type: nauc_precision_at_1_std - value: 9.139550923680906 - - type: nauc_precision_at_20_diff1 - value: 8.366060991850837 - - type: nauc_precision_at_20_max - value: 12.54677453818851 - - type: nauc_precision_at_20_std - value: 20.107820078217852 - - type: nauc_precision_at_3_diff1 - value: 8.338749776595783 - - type: nauc_precision_at_3_max - value: 14.63768301600882 - - type: nauc_precision_at_3_std - value: 11.33362167066395 - - type: nauc_precision_at_5_diff1 - value: 6.704572848424477 - - type: nauc_precision_at_5_max - value: 14.248151818628482 - - type: nauc_precision_at_5_std - value: 13.691110824662891 - - type: nauc_recall_at_1000_diff1 - value: 8.89266377200224 - - type: nauc_recall_at_1000_max - value: 23.61464707731056 - - type: nauc_recall_at_1000_std - value: 48.60798306825727 - - type: nauc_recall_at_100_diff1 - value: 9.377072086689383 - - type: nauc_recall_at_100_max - value: 15.775682252417747 - - type: nauc_recall_at_100_std - value: 33.86046204385918 - - type: nauc_recall_at_10_diff1 - value: 4.098313364685181 - - type: nauc_recall_at_10_max - value: 14.716720941128566 - - type: nauc_recall_at_10_std - value: 20.282626139543357 - - type: nauc_recall_at_1_diff1 - value: 19.07356377322629 - - type: nauc_recall_at_1_max - value: 17.999572753995082 - - type: nauc_recall_at_1_std - value: 10.302044048546005 - - type: nauc_recall_at_20_diff1 - value: 4.393976466517198 - - type: nauc_recall_at_20_max - value: 11.95178800677615 - - type: nauc_recall_at_20_std - value: 20.557023701011847 - - type: nauc_recall_at_3_diff1 - value: 6.734442526385427 - - type: nauc_recall_at_3_max - value: 14.450825242271362 - - type: nauc_recall_at_3_std - value: 11.589452715632738 - - type: nauc_recall_at_5_diff1 - value: 4.762756159503797 - - type: nauc_recall_at_5_max - value: 13.251540766745999 - - type: nauc_recall_at_5_std - value: 12.893569387731784 - - type: ndcg_at_1 - value: 12.2 - - type: ndcg_at_10 - value: 21.26 - - type: ndcg_at_100 - value: 26.913999999999998 - - type: ndcg_at_1000 - value: 30.043999999999997 - - type: ndcg_at_20 - value: 23.145 - - type: ndcg_at_3 - value: 16.758 - - type: ndcg_at_5 - value: 18.715 - - type: precision_at_1 - value: 12.2 - - type: precision_at_10 - value: 3.789 - - type: precision_at_100 - value: 0.719 - - type: precision_at_1000 - value: 0.106 - - type: precision_at_20 - value: 2.338 - - type: precision_at_3 - value: 7.64 - - type: precision_at_5 - value: 5.7299999999999995 - - type: recall_at_1 - value: 10.974 - - type: recall_at_10 - value: 32.634 - - type: recall_at_100 - value: 59.209999999999994 - - type: recall_at_1000 - value: 83.458 - - type: recall_at_20 - value: 39.928999999999995 - - type: recall_at_3 - value: 20.283 - - type: recall_at_5 - value: 24.997 - task: - type: Retrieval - - dataset: - config: default - name: MTEB ClimateFEVER (default) - revision: 47f2ac6acb640fc46020b02a5b59fdda04d39380 - split: test - type: mteb/climate-fever - metrics: - - type: main_score - value: 14.97 - - type: map_at_1 - value: 4.928 - - type: map_at_10 - value: 9.67 - - type: map_at_100 - value: 11.131 - - type: map_at_1000 - value: 11.361 - - type: map_at_20 - value: 10.248 - - type: map_at_3 - value: 7.931000000000001 - - type: map_at_5 - value: 8.780000000000001 - - type: mrr_at_1 - value: 11.856677524429967 - - type: mrr_at_10 - value: 20.35530737810866 - - type: mrr_at_100 - value: 21.828454101212888 - - type: mrr_at_1000 - value: 21.890691227973136 - - type: mrr_at_20 - value: 21.14806585505244 - - type: mrr_at_3 - value: 17.730727470141158 - - type: mrr_at_5 - value: 19.10206297502713 - - type: nauc_map_at_1000_diff1 - value: 7.343974864659536 - - type: nauc_map_at_1000_max - value: 27.127970014906786 - - type: nauc_map_at_1000_std - value: 18.688546971841117 - - type: nauc_map_at_100_diff1 - value: 7.257359641312383 - - type: nauc_map_at_100_max - value: 26.93386501878897 - - type: nauc_map_at_100_std - value: 18.572058476864395 - - type: nauc_map_at_10_diff1 - value: 7.5524511559283285 - - type: nauc_map_at_10_max - value: 25.5823403439562 - - type: nauc_map_at_10_std - value: 16.127814097430974 - - type: nauc_map_at_1_diff1 - value: 12.999418845432006 - - type: nauc_map_at_1_max - value: 31.020337243647095 - - type: nauc_map_at_1_std - value: 19.991896333019515 - - type: nauc_map_at_20_diff1 - value: 7.085975699404192 - - type: nauc_map_at_20_max - value: 25.75407679129859 - - type: nauc_map_at_20_std - value: 16.72526052571226 - - type: nauc_map_at_3_diff1 - value: 8.70936707305993 - - type: nauc_map_at_3_max - value: 26.708517715732246 - - type: nauc_map_at_3_std - value: 15.760631791317781 - - type: nauc_map_at_5_diff1 - value: 8.26315708475148 - - type: nauc_map_at_5_max - value: 25.35727921546233 - - type: nauc_map_at_5_std - value: 15.092258215857902 - - type: nauc_mrr_at_1000_diff1 - value: 9.605823313934675 - - type: nauc_mrr_at_1000_max - value: 29.210900493669495 - - type: nauc_mrr_at_1000_std - value: 18.83080719056213 - - type: nauc_mrr_at_100_diff1 - value: 9.55666513233699 - - type: nauc_mrr_at_100_max - value: 29.22087280589394 - - type: nauc_mrr_at_100_std - value: 18.887436425893338 - - type: nauc_mrr_at_10_diff1 - value: 9.536442166623397 - - type: nauc_mrr_at_10_max - value: 28.489301904218312 - - type: nauc_mrr_at_10_std - value: 17.945183230798516 - - type: nauc_mrr_at_1_diff1 - value: 15.28049719426095 - - type: nauc_mrr_at_1_max - value: 34.27049723189518 - - type: nauc_mrr_at_1_std - value: 21.901296884315475 - - type: nauc_mrr_at_20_diff1 - value: 9.416526110965544 - - type: nauc_mrr_at_20_max - value: 28.75113126415786 - - type: nauc_mrr_at_20_std - value: 18.345484239730165 - - type: nauc_mrr_at_3_diff1 - value: 10.440304488887639 - - type: nauc_mrr_at_3_max - value: 29.419182241937804 - - type: nauc_mrr_at_3_std - value: 17.79516933724722 - - type: nauc_mrr_at_5_diff1 - value: 9.933840128353966 - - type: nauc_mrr_at_5_max - value: 28.38615077411181 - - type: nauc_mrr_at_5_std - value: 17.56861196368992 - - type: nauc_ndcg_at_1000_diff1 - value: 7.4723313593794565 - - type: nauc_ndcg_at_1000_max - value: 31.639339374755142 - - type: nauc_ndcg_at_1000_std - value: 25.105356776676658 - - type: nauc_ndcg_at_100_diff1 - value: 6.308931462956051 - - type: nauc_ndcg_at_100_max - value: 30.655011045261972 - - type: nauc_ndcg_at_100_std - value: 25.33133541833919 - - type: nauc_ndcg_at_10_diff1 - value: 6.318892234859429 - - type: nauc_ndcg_at_10_max - value: 24.70336979572261 - - type: nauc_ndcg_at_10_std - value: 16.19018272099447 - - type: nauc_ndcg_at_1_diff1 - value: 15.28049719426095 - - type: nauc_ndcg_at_1_max - value: 34.27049723189518 - - type: nauc_ndcg_at_1_std - value: 21.901296884315475 - - type: nauc_ndcg_at_20_diff1 - value: 5.361859199429646 - - type: nauc_ndcg_at_20_max - value: 25.400570204198896 - - type: nauc_ndcg_at_20_std - value: 17.887844183290667 - - type: nauc_ndcg_at_3_diff1 - value: 8.110953561431906 - - type: nauc_ndcg_at_3_max - value: 26.63953855972143 - - type: nauc_ndcg_at_3_std - value: 15.385397581839417 - - type: nauc_ndcg_at_5_diff1 - value: 7.709924887062054 - - type: nauc_ndcg_at_5_max - value: 24.490998203146383 - - type: nauc_ndcg_at_5_std - value: 14.6921440386966 - - type: nauc_precision_at_1000_diff1 - value: 3.6765328094092595 - - type: nauc_precision_at_1000_max - value: 22.330333946150965 - - type: nauc_precision_at_1000_std - value: 22.933547000215604 - - type: nauc_precision_at_100_diff1 - value: 2.2666175638162382 - - type: nauc_precision_at_100_max - value: 30.354739309785444 - - type: nauc_precision_at_100_std - value: 33.562087725077596 - - type: nauc_precision_at_10_diff1 - value: 1.8399284347123332 - - type: nauc_precision_at_10_max - value: 21.55462109276507 - - type: nauc_precision_at_10_std - value: 16.121481123984722 - - type: nauc_precision_at_1_diff1 - value: 15.28049719426095 - - type: nauc_precision_at_1_max - value: 34.27049723189518 - - type: nauc_precision_at_1_std - value: 21.901296884315475 - - type: nauc_precision_at_20_diff1 - value: 0.019989740402583465 - - type: nauc_precision_at_20_max - value: 22.184928806584196 - - type: nauc_precision_at_20_std - value: 19.352421167092725 - - type: nauc_precision_at_3_diff1 - value: 4.993655109383327 - - type: nauc_precision_at_3_max - value: 25.471904019950102 - - type: nauc_precision_at_3_std - value: 14.428827762549512 - - type: nauc_precision_at_5_diff1 - value: 4.356666010145262 - - type: nauc_precision_at_5_max - value: 21.93208543594649 - - type: nauc_precision_at_5_std - value: 13.168742223430959 - - type: nauc_recall_at_1000_diff1 - value: 7.275822779465287 - - type: nauc_recall_at_1000_max - value: 34.71656818418004 - - type: nauc_recall_at_1000_std - value: 34.358417311322675 - - type: nauc_recall_at_100_diff1 - value: 2.6813944416625026 - - type: nauc_recall_at_100_max - value: 30.063206974414825 - - type: nauc_recall_at_100_std - value: 32.997643399627634 - - type: nauc_recall_at_10_diff1 - value: 2.049411072425148 - - type: nauc_recall_at_10_max - value: 17.384327223968484 - - type: nauc_recall_at_10_std - value: 12.313062153156563 - - type: nauc_recall_at_1_diff1 - value: 12.999418845432006 - - type: nauc_recall_at_1_max - value: 31.020337243647095 - - type: nauc_recall_at_1_std - value: 19.991896333019515 - - type: nauc_recall_at_20_diff1 - value: 0.0769739475464366 - - type: nauc_recall_at_20_max - value: 18.40025074677139 - - type: nauc_recall_at_20_std - value: 15.71519209254166 - - type: nauc_recall_at_3_diff1 - value: 5.89259927853003 - - type: nauc_recall_at_3_max - value: 21.39147480034604 - - type: nauc_recall_at_3_std - value: 11.573847953419673 - - type: nauc_recall_at_5_diff1 - value: 4.925250715081651 - - type: nauc_recall_at_5_max - value: 17.681821165181706 - - type: nauc_recall_at_5_std - value: 10.016468241533516 - - type: ndcg_at_1 - value: 11.857 - - type: ndcg_at_10 - value: 14.97 - - type: ndcg_at_100 - value: 22.719 - - type: ndcg_at_1000 - value: 27.034000000000002 - - type: ndcg_at_20 - value: 17.092 - - type: ndcg_at_3 - value: 11.584 - - type: ndcg_at_5 - value: 12.715000000000002 - - type: precision_at_1 - value: 11.857 - - type: precision_at_10 - value: 4.8340000000000005 - - type: precision_at_100 - value: 1.336 - - type: precision_at_1000 - value: 0.212 - - type: precision_at_20 - value: 3.309 - - type: precision_at_3 - value: 8.903 - - type: precision_at_5 - value: 6.84 - - type: recall_at_1 - value: 4.928 - - type: recall_at_10 - value: 19.852 - - type: recall_at_100 - value: 48.075 - - type: recall_at_1000 - value: 72.60799999999999 - - type: recall_at_20 - value: 25.936999999999998 - - type: recall_at_3 - value: 11.309 - - type: recall_at_5 - value: 14.567 - task: - type: Retrieval - - dataset: - config: default - name: MTEB DBPedia (default) - revision: c0f706b76e590d620bd6618b3ca8efdd34e2d659 - split: test - type: mteb/dbpedia - metrics: - - type: main_score - value: 30.137000000000004 - - type: map_at_1 - value: 6.0 - - type: map_at_10 - value: 13.453000000000001 - - type: map_at_100 - value: 18.389 - - type: map_at_1000 - value: 19.706000000000003 - - type: map_at_20 - value: 15.348 - - type: map_at_3 - value: 9.569999999999999 - - type: map_at_5 - value: 11.269 - - type: mrr_at_1 - value: 48.5 - - type: mrr_at_10 - value: 59.12063492063492 - - type: mrr_at_100 - value: 59.7067363969307 - - type: mrr_at_1000 - value: 59.73195255745748 - - type: mrr_at_20 - value: 59.49264190079787 - - type: mrr_at_3 - value: 56.79166666666667 - - type: mrr_at_5 - value: 58.10416666666667 - - type: nauc_map_at_1000_diff1 - value: 21.11048962823763 - - type: nauc_map_at_1000_max - value: 3.824918831917913 - - type: nauc_map_at_1000_std - value: 18.091642998292183 - - type: nauc_map_at_100_diff1 - value: 21.400896998453565 - - type: nauc_map_at_100_max - value: 3.0532938872682527 - - type: nauc_map_at_100_std - value: 15.748553761470577 - - type: nauc_map_at_10_diff1 - value: 21.466273522809004 - - type: nauc_map_at_10_max - value: 1.0436684944087984 - - type: nauc_map_at_10_std - value: 3.123980868343808 - - type: nauc_map_at_1_diff1 - value: 36.294956943711185 - - type: nauc_map_at_1_max - value: 5.187621118221234 - - type: nauc_map_at_1_std - value: -2.793466227392795 - - type: nauc_map_at_20_diff1 - value: 22.487809942004276 - - type: nauc_map_at_20_max - value: 1.810008116855991 - - type: nauc_map_at_20_std - value: 6.582415021321395 - - type: nauc_map_at_3_diff1 - value: 23.65507925300577 - - type: nauc_map_at_3_max - value: 1.5046652580013493 - - type: nauc_map_at_3_std - value: -0.7516301806524871 - - type: nauc_map_at_5_diff1 - value: 21.46128727551178 - - type: nauc_map_at_5_max - value: 1.382221300969405 - - type: nauc_map_at_5_std - value: 0.5899543778583619 - - type: nauc_mrr_at_1000_diff1 - value: 39.95976660401341 - - type: nauc_mrr_at_1000_max - value: 14.926325657707467 - - type: nauc_mrr_at_1000_std - value: 16.345761016463396 - - type: nauc_mrr_at_100_diff1 - value: 39.959045716548836 - - type: nauc_mrr_at_100_max - value: 14.934953971746829 - - type: nauc_mrr_at_100_std - value: 16.376010454801467 - - type: nauc_mrr_at_10_diff1 - value: 40.06090715779124 - - type: nauc_mrr_at_10_max - value: 14.881820498473369 - - type: nauc_mrr_at_10_std - value: 16.59218049651554 - - type: nauc_mrr_at_1_diff1 - value: 40.879388159507975 - - type: nauc_mrr_at_1_max - value: 11.270536619634298 - - type: nauc_mrr_at_1_std - value: 13.0413747854383 - - type: nauc_mrr_at_20_diff1 - value: 40.01647777717407 - - type: nauc_mrr_at_20_max - value: 14.937806497015416 - - type: nauc_mrr_at_20_std - value: 16.472071311966392 - - type: nauc_mrr_at_3_diff1 - value: 39.39281882829406 - - type: nauc_mrr_at_3_max - value: 14.321953204624844 - - type: nauc_mrr_at_3_std - value: 13.798281679964585 - - type: nauc_mrr_at_5_diff1 - value: 39.768973978552594 - - type: nauc_mrr_at_5_max - value: 14.404530882907713 - - type: nauc_mrr_at_5_std - value: 15.92346805944451 - - type: nauc_ndcg_at_1000_diff1 - value: 22.27822653984692 - - type: nauc_ndcg_at_1000_max - value: 9.924226549383965 - - type: nauc_ndcg_at_1000_std - value: 28.25033871964479 - - type: nauc_ndcg_at_100_diff1 - value: 23.50078106583016 - - type: nauc_ndcg_at_100_max - value: 6.7543474842024604 - - type: nauc_ndcg_at_100_std - value: 21.003192623211657 - - type: nauc_ndcg_at_10_diff1 - value: 26.804949345594103 - - type: nauc_ndcg_at_10_max - value: 5.287967810437971 - - type: nauc_ndcg_at_10_std - value: 15.013863454844003 - - type: nauc_ndcg_at_1_diff1 - value: 36.51486006881417 - - type: nauc_ndcg_at_1_max - value: 10.405773346698801 - - type: nauc_ndcg_at_1_std - value: 11.799158787353369 - - type: nauc_ndcg_at_20_diff1 - value: 27.538405701690365 - - type: nauc_ndcg_at_20_max - value: 5.803911184640402 - - type: nauc_ndcg_at_20_std - value: 14.736490152112392 - - type: nauc_ndcg_at_3_diff1 - value: 27.617656142008574 - - type: nauc_ndcg_at_3_max - value: 8.495506842062914 - - type: nauc_ndcg_at_3_std - value: 13.000529342799002 - - type: nauc_ndcg_at_5_diff1 - value: 26.11786480923478 - - type: nauc_ndcg_at_5_max - value: 6.810483365982102 - - type: nauc_ndcg_at_5_std - value: 13.686050622685887 - - type: nauc_precision_at_1000_diff1 - value: -1.8263159519304677 - - type: nauc_precision_at_1000_max - value: 9.105654485044203 - - type: nauc_precision_at_1000_std - value: 19.896196956194387 - - type: nauc_precision_at_100_diff1 - value: 5.421790848083438 - - type: nauc_precision_at_100_max - value: 5.843754140943668 - - type: nauc_precision_at_100_std - value: 32.872530636176 - - type: nauc_precision_at_10_diff1 - value: 14.266901547255353 - - type: nauc_precision_at_10_max - value: 5.6720582438158935 - - type: nauc_precision_at_10_std - value: 24.731755691337355 - - type: nauc_precision_at_1_diff1 - value: 40.879388159507975 - - type: nauc_precision_at_1_max - value: 11.270536619634298 - - type: nauc_precision_at_1_std - value: 13.0413747854383 - - type: nauc_precision_at_20_diff1 - value: 14.561505270649377 - - type: nauc_precision_at_20_max - value: 6.1877072435924365 - - type: nauc_precision_at_20_std - value: 28.164304475344604 - - type: nauc_precision_at_3_diff1 - value: 20.90355054405486 - - type: nauc_precision_at_3_max - value: 8.234045919189285 - - type: nauc_precision_at_3_std - value: 15.882744817436434 - - type: nauc_precision_at_5_diff1 - value: 16.42363926772338 - - type: nauc_precision_at_5_max - value: 6.6904533726502695 - - type: nauc_precision_at_5_std - value: 19.664135643199458 - - type: nauc_recall_at_1000_diff1 - value: 5.50000056743861 - - type: nauc_recall_at_1000_max - value: 6.615548983596191 - - type: nauc_recall_at_1000_std - value: 33.480962113017895 - - type: nauc_recall_at_100_diff1 - value: 9.888973775211436 - - type: nauc_recall_at_100_max - value: 2.7176483244916687 - - type: nauc_recall_at_100_std - value: 20.844187088199874 - - type: nauc_recall_at_10_diff1 - value: 16.13403101219931 - - type: nauc_recall_at_10_max - value: -0.4718490485814091 - - type: nauc_recall_at_10_std - value: 2.1253602607327875 - - type: nauc_recall_at_1_diff1 - value: 36.294956943711185 - - type: nauc_recall_at_1_max - value: 5.187621118221234 - - type: nauc_recall_at_1_std - value: -2.793466227392795 - - type: nauc_recall_at_20_diff1 - value: 19.48409505091461 - - type: nauc_recall_at_20_max - value: 3.2951687936987093 - - type: nauc_recall_at_20_std - value: 7.00681132960327 - - type: nauc_recall_at_3_diff1 - value: 18.267617168970432 - - type: nauc_recall_at_3_max - value: -0.06449790862866218 - - type: nauc_recall_at_3_std - value: -3.016158705432899 - - type: nauc_recall_at_5_diff1 - value: 14.913123369125838 - - type: nauc_recall_at_5_max - value: 0.16850104026158685 - - type: nauc_recall_at_5_std - value: -0.6034095216457759 - - type: ndcg_at_1 - value: 38.5 - - type: ndcg_at_10 - value: 30.137000000000004 - - type: ndcg_at_100 - value: 34.342 - - type: ndcg_at_1000 - value: 42.201 - - type: ndcg_at_20 - value: 29.822 - - type: ndcg_at_3 - value: 32.983000000000004 - - type: ndcg_at_5 - value: 31.182 - - type: precision_at_1 - value: 48.5 - - type: precision_at_10 - value: 24.099999999999998 - - type: precision_at_100 - value: 7.605 - - type: precision_at_1000 - value: 1.733 - - type: precision_at_20 - value: 18.137 - - type: precision_at_3 - value: 36.083 - - type: precision_at_5 - value: 30.75 - - type: recall_at_1 - value: 6.0 - - type: recall_at_10 - value: 19.136 - - type: recall_at_100 - value: 41.338 - - type: recall_at_1000 - value: 67.25 - - type: recall_at_20 - value: 24.688 - - type: recall_at_3 - value: 11.132 - - type: recall_at_5 - value: 14.231 - task: - type: Retrieval - - dataset: - config: default - name: MTEB EmotionClassification (default) - revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37 - split: test - type: mteb/emotion - metrics: - - type: accuracy - value: 73.09500000000001 - - type: f1 - value: 68.63641879623752 - - type: f1_weighted - value: 74.08687647970862 - - type: main_score - value: 73.09500000000001 - task: - type: Classification - - dataset: - config: default - name: MTEB FEVER (default) - revision: bea83ef9e8fb933d90a2f1d5515737465d613e12 - split: test - type: mteb/fever - metrics: - - type: main_score - value: 56.275 - - type: map_at_1 - value: 35.847 - - type: map_at_10 - value: 49.192 - - type: map_at_100 - value: 49.913000000000004 - - type: map_at_1000 - value: 49.945 - - type: map_at_20 - value: 49.652 - - type: map_at_3 - value: 45.988 - - type: map_at_5 - value: 47.955 - - type: mrr_at_1 - value: 38.5988598859886 - - type: mrr_at_10 - value: 52.23110406278746 - - type: mrr_at_100 - value: 52.88492739273575 - - type: mrr_at_1000 - value: 52.904747106357384 - - type: mrr_at_20 - value: 52.6636924268024 - - type: mrr_at_3 - value: 48.97239723972427 - - type: mrr_at_5 - value: 51.01635163516368 - - type: nauc_map_at_1000_diff1 - value: 36.64174970870812 - - type: nauc_map_at_1000_max - value: 15.660507126063871 - - type: nauc_map_at_1000_std - value: -12.525150655309774 - - type: nauc_map_at_100_diff1 - value: 36.6435791136361 - - type: nauc_map_at_100_max - value: 15.675834344014946 - - type: nauc_map_at_100_std - value: -12.504087019254898 - - type: nauc_map_at_10_diff1 - value: 36.512625030119366 - - type: nauc_map_at_10_max - value: 15.521396090488226 - - type: nauc_map_at_10_std - value: -12.655309941517057 - - type: nauc_map_at_1_diff1 - value: 38.19401754151657 - - type: nauc_map_at_1_max - value: 13.290047228169902 - - type: nauc_map_at_1_std - value: -16.25017425222326 - - type: nauc_map_at_20_diff1 - value: 36.61973919620995 - - type: nauc_map_at_20_max - value: 15.679871911858962 - - type: nauc_map_at_20_std - value: -12.487736624487598 - - type: nauc_map_at_3_diff1 - value: 36.64249275169512 - - type: nauc_map_at_3_max - value: 14.555494484309506 - - type: nauc_map_at_3_std - value: -14.778136642316362 - - type: nauc_map_at_5_diff1 - value: 36.60147619857385 - - type: nauc_map_at_5_max - value: 15.091571640821227 - - type: nauc_map_at_5_std - value: -13.570700587891283 - - type: nauc_mrr_at_1000_diff1 - value: 37.997426431979406 - - type: nauc_mrr_at_1000_max - value: 14.98013800807032 - - type: nauc_mrr_at_1000_std - value: -14.841942787676668 - - type: nauc_mrr_at_100_diff1 - value: 38.00076769617504 - - type: nauc_mrr_at_100_max - value: 14.996219532747034 - - type: nauc_mrr_at_100_std - value: -14.814824444455867 - - type: nauc_mrr_at_10_diff1 - value: 37.80491807703189 - - type: nauc_mrr_at_10_max - value: 14.882959946615573 - - type: nauc_mrr_at_10_std - value: -14.87210289207 - - type: nauc_mrr_at_1_diff1 - value: 39.30857248856019 - - type: nauc_mrr_at_1_max - value: 13.216181318724654 - - type: nauc_mrr_at_1_std - value: -17.816031667670963 - - type: nauc_mrr_at_20_diff1 - value: 37.958327539289535 - - type: nauc_mrr_at_20_max - value: 15.01939060882702 - - type: nauc_mrr_at_20_std - value: -14.742321408684889 - - type: nauc_mrr_at_3_diff1 - value: 37.83776099785701 - - type: nauc_mrr_at_3_max - value: 13.991066447397454 - - type: nauc_mrr_at_3_std - value: -16.852370243294917 - - type: nauc_mrr_at_5_diff1 - value: 37.80241399940709 - - type: nauc_mrr_at_5_max - value: 14.504998552455787 - - type: nauc_mrr_at_5_std - value: -15.653347947582471 - - type: nauc_ndcg_at_1000_diff1 - value: 36.65756580123748 - - type: nauc_ndcg_at_1000_max - value: 16.97444748484926 - - type: nauc_ndcg_at_1000_std - value: -9.948608541675865 - - type: nauc_ndcg_at_100_diff1 - value: 36.71481365126713 - - type: nauc_ndcg_at_100_max - value: 17.489332769921877 - - type: nauc_ndcg_at_100_std - value: -9.11867227279014 - - type: nauc_ndcg_at_10_diff1 - value: 36.06799961534314 - - type: nauc_ndcg_at_10_max - value: 16.8613677000495 - - type: nauc_ndcg_at_10_std - value: -9.81556106125859 - - type: nauc_ndcg_at_1_diff1 - value: 39.30857248856019 - - type: nauc_ndcg_at_1_max - value: 13.216181318724654 - - type: nauc_ndcg_at_1_std - value: -17.816031667670963 - - type: nauc_ndcg_at_20_diff1 - value: 36.49965813253492 - - type: nauc_ndcg_at_20_max - value: 17.539102339776296 - - type: nauc_ndcg_at_20_std - value: -9.026565538786357 - - type: nauc_ndcg_at_3_diff1 - value: 36.41815839693589 - - type: nauc_ndcg_at_3_max - value: 14.700854622435058 - - type: nauc_ndcg_at_3_std - value: -14.674234384559073 - - type: nauc_ndcg_at_5_diff1 - value: 36.28943043198077 - - type: nauc_ndcg_at_5_max - value: 15.745879382099279 - - type: nauc_ndcg_at_5_std - value: -12.24120367991547 - - type: nauc_precision_at_1000_diff1 - value: -1.125165071294792 - - type: nauc_precision_at_1000_max - value: 6.995559652372453 - - type: nauc_precision_at_1000_std - value: 12.831131017541821 - - type: nauc_precision_at_100_diff1 - value: 11.000701192784645 - - type: nauc_precision_at_100_max - value: 20.360887483968636 - - type: nauc_precision_at_100_std - value: 22.16846897480723 - - type: nauc_precision_at_10_diff1 - value: 27.00078852057367 - - type: nauc_precision_at_10_max - value: 21.829143356067338 - - type: nauc_precision_at_10_std - value: 5.798370950077175 - - type: nauc_precision_at_1_diff1 - value: 39.30857248856019 - - type: nauc_precision_at_1_max - value: 13.216181318724654 - - type: nauc_precision_at_1_std - value: -17.816031667670963 - - type: nauc_precision_at_20_diff1 - value: 24.362454958286268 - - type: nauc_precision_at_20_max - value: 25.723944313836018 - - type: nauc_precision_at_20_std - value: 14.965719248393347 - - type: nauc_precision_at_3_diff1 - value: 33.798877367578854 - - type: nauc_precision_at_3_max - value: 15.363655428823531 - - type: nauc_precision_at_3_std - value: -13.791082916394782 - - type: nauc_precision_at_5_diff1 - value: 31.956562731570497 - - type: nauc_precision_at_5_max - value: 18.233368709013877 - - type: nauc_precision_at_5_std - value: -6.431537681620625 - - type: nauc_recall_at_1000_diff1 - value: 18.340052781682274 - - type: nauc_recall_at_1000_max - value: 31.963275437350024 - - type: nauc_recall_at_1000_std - value: 44.816169836161656 - - type: nauc_recall_at_100_diff1 - value: 28.22528558216003 - - type: nauc_recall_at_100_max - value: 34.5813678838198 - - type: nauc_recall_at_100_std - value: 34.67537364886928 - - type: nauc_recall_at_10_diff1 - value: 29.7947859758469 - - type: nauc_recall_at_10_max - value: 22.49573322191089 - - type: nauc_recall_at_10_std - value: 7.132599572662985 - - type: nauc_recall_at_1_diff1 - value: 38.19401754151657 - - type: nauc_recall_at_1_max - value: 13.290047228169902 - - type: nauc_recall_at_1_std - value: -16.25017425222326 - - type: nauc_recall_at_20_diff1 - value: 30.26074789910862 - - type: nauc_recall_at_20_max - value: 27.834226251757816 - - type: nauc_recall_at_20_std - value: 16.54841199015065 - - type: nauc_recall_at_3_diff1 - value: 33.413742439395484 - - type: nauc_recall_at_3_max - value: 15.211407121882475 - - type: nauc_recall_at_3_std - value: -11.671768186203062 - - type: nauc_recall_at_5_diff1 - value: 32.21235453464061 - - type: nauc_recall_at_5_max - value: 17.82957773085371 - - type: nauc_recall_at_5_std - value: -4.588112602146949 - - type: ndcg_at_1 - value: 38.599 - - type: ndcg_at_10 - value: 56.275 - - type: ndcg_at_100 - value: 59.580999999999996 - - type: ndcg_at_1000 - value: 60.307 - - type: ndcg_at_20 - value: 57.865 - - type: ndcg_at_3 - value: 49.903 - - type: ndcg_at_5 - value: 53.378 - - type: precision_at_1 - value: 38.599 - - type: precision_at_10 - value: 8.237 - - type: precision_at_100 - value: 1.004 - - type: precision_at_1000 - value: 0.108 - - type: precision_at_20 - value: 4.47 - - type: precision_at_3 - value: 20.982 - - type: precision_at_5 - value: 14.475999999999999 - - type: recall_at_1 - value: 35.847 - - type: recall_at_10 - value: 75.244 - - type: recall_at_100 - value: 89.968 - - type: recall_at_1000 - value: 95.281 - - type: recall_at_20 - value: 81.272 - - type: recall_at_3 - value: 58.148999999999994 - - type: recall_at_5 - value: 66.53 - task: - type: Retrieval - - dataset: - config: default - name: MTEB FiQA2018 (default) - revision: 27a168819829fe9bcd655c2df245fb19452e8e06 - split: test - type: mteb/fiqa - metrics: - - type: main_score - value: 73.56 - - type: map_at_1 - value: 36.149 - - type: map_at_10 - value: 64.262 - - type: map_at_100 - value: 66.346 - - type: map_at_1000 - value: 66.381 - - type: map_at_20 - value: 65.91 - - type: map_at_3 - value: 54.579 - - type: map_at_5 - value: 60.043 - - type: mrr_at_1 - value: 68.20987654320987 - - type: mrr_at_10 - value: 78.0676195375269 - - type: mrr_at_100 - value: 78.22788481128512 - - type: mrr_at_1000 - value: 78.2351762841002 - - type: mrr_at_20 - value: 78.19418362303348 - - type: mrr_at_3 - value: 76.38888888888886 - - type: mrr_at_5 - value: 77.53858024691351 - - type: nauc_map_at_1000_diff1 - value: 39.13951115792765 - - type: nauc_map_at_1000_max - value: 15.789675641496808 - - type: nauc_map_at_1000_std - value: -14.962767430758966 - - type: nauc_map_at_100_diff1 - value: 39.14667307112649 - - type: nauc_map_at_100_max - value: 15.7556118076197 - - type: nauc_map_at_100_std - value: -14.947192943164422 - - type: nauc_map_at_10_diff1 - value: 39.71365566339051 - - type: nauc_map_at_10_max - value: 12.359982700266224 - - type: nauc_map_at_10_std - value: -16.689077406647765 - - type: nauc_map_at_1_diff1 - value: 38.01390755497353 - - type: nauc_map_at_1_max - value: -2.4285798001758696 - - type: nauc_map_at_1_std - value: -11.839428013172745 - - type: nauc_map_at_20_diff1 - value: 39.27021403147416 - - type: nauc_map_at_20_max - value: 15.22857604958874 - - type: nauc_map_at_20_std - value: -15.31472460777032 - - type: nauc_map_at_3_diff1 - value: 37.977476069647196 - - type: nauc_map_at_3_max - value: 3.2190612996608228 - - type: nauc_map_at_3_std - value: -15.66245768446722 - - type: nauc_map_at_5_diff1 - value: 39.4541015155147 - - type: nauc_map_at_5_max - value: 8.406851080721177 - - type: nauc_map_at_5_std - value: -17.318320201143724 - - type: nauc_mrr_at_1000_diff1 - value: 50.52179630360454 - - type: nauc_mrr_at_1000_max - value: 27.958043634255976 - - type: nauc_mrr_at_1000_std - value: -10.449298885386217 - - type: nauc_mrr_at_100_diff1 - value: 50.51660811731795 - - type: nauc_mrr_at_100_max - value: 27.944402723175603 - - type: nauc_mrr_at_100_std - value: -10.44533164401012 - - type: nauc_mrr_at_10_diff1 - value: 50.43596746653215 - - type: nauc_mrr_at_10_max - value: 27.849753656390963 - - type: nauc_mrr_at_10_std - value: -10.448303695173873 - - type: nauc_mrr_at_1_diff1 - value: 52.29547373761976 - - type: nauc_mrr_at_1_max - value: 28.84554542629197 - - type: nauc_mrr_at_1_std - value: -9.753666425872645 - - type: nauc_mrr_at_20_diff1 - value: 50.47556830247787 - - type: nauc_mrr_at_20_max - value: 27.92615740500234 - - type: nauc_mrr_at_20_std - value: -10.413757833872396 - - type: nauc_mrr_at_3_diff1 - value: 50.30231795753932 - - type: nauc_mrr_at_3_max - value: 27.23788784979687 - - type: nauc_mrr_at_3_std - value: -11.52516432450595 - - type: nauc_mrr_at_5_diff1 - value: 49.78580615169249 - - type: nauc_mrr_at_5_max - value: 28.46933101763155 - - type: nauc_mrr_at_5_std - value: -10.669870915511963 - - type: nauc_ndcg_at_1000_diff1 - value: 41.21630766413613 - - type: nauc_ndcg_at_1000_max - value: 21.898896244340968 - - type: nauc_ndcg_at_1000_std - value: -11.887919577706999 - - type: nauc_ndcg_at_100_diff1 - value: 41.31739592194364 - - type: nauc_ndcg_at_100_max - value: 21.604565329954767 - - type: nauc_ndcg_at_100_std - value: -11.487948873516627 - - type: nauc_ndcg_at_10_diff1 - value: 41.478136363299946 - - type: nauc_ndcg_at_10_max - value: 16.2617184983789 - - type: nauc_ndcg_at_10_std - value: -14.885574869807709 - - type: nauc_ndcg_at_1_diff1 - value: 52.29547373761976 - - type: nauc_ndcg_at_1_max - value: 28.84554542629197 - - type: nauc_ndcg_at_1_std - value: -9.753666425872645 - - type: nauc_ndcg_at_20_diff1 - value: 41.0669327481738 - - type: nauc_ndcg_at_20_max - value: 20.514680273626567 - - type: nauc_ndcg_at_20_std - value: -12.435494727134182 - - type: nauc_ndcg_at_3_diff1 - value: 36.83325361853425 - - type: nauc_ndcg_at_3_max - value: 17.38656549225942 - - type: nauc_ndcg_at_3_std - value: -13.019339397502755 - - type: nauc_ndcg_at_5_diff1 - value: 38.580162281647056 - - type: nauc_ndcg_at_5_max - value: 16.337906272293548 - - type: nauc_ndcg_at_5_std - value: -15.684185539933097 - - type: nauc_precision_at_1000_diff1 - value: -15.015876704383984 - - type: nauc_precision_at_1000_max - value: 31.597455975629103 - - type: nauc_precision_at_1000_std - value: 10.381684525984323 - - type: nauc_precision_at_100_diff1 - value: -13.999569979209717 - - type: nauc_precision_at_100_max - value: 32.07381175031831 - - type: nauc_precision_at_100_std - value: 10.894062912630725 - - type: nauc_precision_at_10_diff1 - value: -6.838189677762939 - - type: nauc_precision_at_10_max - value: 30.8435231260984 - - type: nauc_precision_at_10_std - value: 3.584893296243004 - - type: nauc_precision_at_1_diff1 - value: 52.29547373761976 - - type: nauc_precision_at_1_max - value: 28.84554542629197 - - type: nauc_precision_at_1_std - value: -9.753666425872645 - - type: nauc_precision_at_20_diff1 - value: -11.288106674270978 - - type: nauc_precision_at_20_max - value: 32.833427715252384 - - type: nauc_precision_at_20_std - value: 8.381049941157823 - - type: nauc_precision_at_3_diff1 - value: 8.995935650409244 - - type: nauc_precision_at_3_max - value: 28.604916357622667 - - type: nauc_precision_at_3_std - value: -3.413011596709042 - - type: nauc_precision_at_5_diff1 - value: 0.5096333185904195 - - type: nauc_precision_at_5_max - value: 31.64022178781689 - - type: nauc_precision_at_5_std - value: -2.229213295556884 - - type: nauc_recall_at_1000_diff1 - value: 11.836577229926256 - - type: nauc_recall_at_1000_max - value: 63.2069044893288 - - type: nauc_recall_at_1000_std - value: 39.48278448112018 - - type: nauc_recall_at_100_diff1 - value: 33.5139879615151 - - type: nauc_recall_at_100_max - value: 22.125177627882646 - - type: nauc_recall_at_100_std - value: 14.591263016360372 - - type: nauc_recall_at_10_diff1 - value: 32.36111021259031 - - type: nauc_recall_at_10_max - value: 1.8443323783196737 - - type: nauc_recall_at_10_std - value: -15.156002529122315 - - type: nauc_recall_at_1_diff1 - value: 38.01390755497353 - - type: nauc_recall_at_1_max - value: -2.4285798001758696 - - type: nauc_recall_at_1_std - value: -11.839428013172745 - - type: nauc_recall_at_20_diff1 - value: 28.09780616502855 - - type: nauc_recall_at_20_max - value: 16.429971434661592 - - type: nauc_recall_at_20_std - value: -2.4852013016206156 - - type: nauc_recall_at_3_diff1 - value: 31.25125120179455 - - type: nauc_recall_at_3_max - value: -3.973563562137874 - - type: nauc_recall_at_3_std - value: -15.354908114563473 - - type: nauc_recall_at_5_diff1 - value: 31.24816861720535 - - type: nauc_recall_at_5_max - value: 2.908586508028948 - - type: nauc_recall_at_5_std - value: -16.221156997636047 - - type: ndcg_at_1 - value: 68.21000000000001 - - type: ndcg_at_10 - value: 73.56 - - type: ndcg_at_100 - value: 77.522 - - type: ndcg_at_1000 - value: 77.95299999999999 - - type: ndcg_at_20 - value: 76.193 - - type: ndcg_at_3 - value: 66.853 - - type: ndcg_at_5 - value: 69.602 - - type: precision_at_1 - value: 68.21000000000001 - - type: precision_at_10 - value: 21.142 - - type: precision_at_100 - value: 2.546 - - type: precision_at_1000 - value: 0.262 - - type: precision_at_20 - value: 11.914 - - type: precision_at_3 - value: 45.525 - - type: precision_at_5 - value: 34.29 - - type: recall_at_1 - value: 36.149 - - type: recall_at_10 - value: 84.102 - - type: recall_at_100 - value: 96.749 - - type: recall_at_1000 - value: 99.34100000000001 - - type: recall_at_20 - value: 91.662 - - type: recall_at_3 - value: 61.602999999999994 - - type: recall_at_5 - value: 72.482 - task: - type: Retrieval - - dataset: - config: default - name: MTEB GeoreviewClassification (default) - revision: 3765c0d1de6b7d264bc459433c45e5a75513839c - split: test - type: ai-forever/georeview-classification - metrics: - - type: accuracy - value: 58.59296875 - - type: f1 - value: 55.87139132757291 - - type: f1_weighted - value: 55.86634332171304 - - type: main_score - value: 58.59296875 - task: - type: Classification - - dataset: - config: default - name: MTEB GeoreviewClusteringP2P (default) - revision: 97a313c8fc85b47f13f33e7e9a95c1ad888c7fec - split: test - type: ai-forever/georeview-clustering-p2p - metrics: - - type: main_score - value: 61.37882703448214 - - type: v_measure - value: 61.37882703448214 - - type: v_measure_std - value: 0.9038440502247551 - task: - type: Clustering - - dataset: - config: default - name: MTEB HeadlineClassification (default) - revision: 2fe05ee6b5832cda29f2ef7aaad7b7fe6a3609eb - split: test - type: ai-forever/headline-classification - metrics: - - type: accuracy - value: 86.01484375 - - type: f1 - value: 85.17323995821744 - - type: f1_weighted - value: 85.17273609511149 - - type: main_score - value: 86.01484375 - task: - type: Classification - - dataset: - config: default - name: MTEB HotpotQA (default) - revision: ab518f4d6fcca38d87c25209f94beba119d02014 - split: test - type: mteb/hotpotqa - metrics: - - type: main_score - value: 56.936 - - type: map_at_1 - value: 32.836 - - type: map_at_10 - value: 47.016000000000005 - - type: map_at_100 - value: 48.047000000000004 - - type: map_at_1000 - value: 48.125 - - type: map_at_20 - value: 47.602 - - type: map_at_3 - value: 43.508 - - type: map_at_5 - value: 45.587 - - type: mrr_at_1 - value: 65.67184334908845 - - type: mrr_at_10 - value: 73.74106406010503 - - type: mrr_at_100 - value: 74.07360674005218 - - type: mrr_at_1000 - value: 74.08528396466417 - - type: mrr_at_20 - value: 73.95171995882409 - - type: mrr_at_3 - value: 72.12469052442032 - - type: mrr_at_5 - value: 73.12401530497377 - - type: nauc_map_at_1000_diff1 - value: 34.005453539011164 - - type: nauc_map_at_1000_max - value: 25.82115616984131 - - type: nauc_map_at_1000_std - value: 1.4158465225486871 - - type: nauc_map_at_100_diff1 - value: 33.97885841853144 - - type: nauc_map_at_100_max - value: 25.788066937953534 - - type: nauc_map_at_100_std - value: 1.4014125260716204 - - type: nauc_map_at_10_diff1 - value: 34.208269490411624 - - type: nauc_map_at_10_max - value: 25.81236166730827 - - type: nauc_map_at_10_std - value: 0.8120958236163661 - - type: nauc_map_at_1_diff1 - value: 69.54414023975188 - - type: nauc_map_at_1_max - value: 40.65799250364921 - - type: nauc_map_at_1_std - value: -5.03532335561708 - - type: nauc_map_at_20_diff1 - value: 34.042140247706996 - - type: nauc_map_at_20_max - value: 25.793308131593257 - - type: nauc_map_at_20_std - value: 1.1211505623511913 - - type: nauc_map_at_3_diff1 - value: 37.20605399174748 - - type: nauc_map_at_3_max - value: 27.418797479842418 - - type: nauc_map_at_3_std - value: -0.7640077998567261 - - type: nauc_map_at_5_diff1 - value: 35.018465182394756 - - type: nauc_map_at_5_max - value: 26.433512678612697 - - type: nauc_map_at_5_std - value: 0.11297600218386353 - - type: nauc_mrr_at_1000_diff1 - value: 67.25936635050807 - - type: nauc_mrr_at_1000_max - value: 41.71491867312844 - - type: nauc_mrr_at_1000_std - value: -3.087319936519033 - - type: nauc_mrr_at_100_diff1 - value: 67.25590240269183 - - type: nauc_mrr_at_100_max - value: 41.71790879024321 - - type: nauc_mrr_at_100_std - value: -3.0740353666702234 - - type: nauc_mrr_at_10_diff1 - value: 67.1799911142862 - - type: nauc_mrr_at_10_max - value: 41.71487539858882 - - type: nauc_mrr_at_10_std - value: -3.0806545113886785 - - type: nauc_mrr_at_1_diff1 - value: 69.54414023975188 - - type: nauc_mrr_at_1_max - value: 40.65799250364921 - - type: nauc_mrr_at_1_std - value: -5.03532335561708 - - type: nauc_mrr_at_20_diff1 - value: 67.22701732432218 - - type: nauc_mrr_at_20_max - value: 41.72352029115392 - - type: nauc_mrr_at_20_std - value: -3.083058785296735 - - type: nauc_mrr_at_3_diff1 - value: 66.96430614667666 - - type: nauc_mrr_at_3_max - value: 41.48604737080645 - - type: nauc_mrr_at_3_std - value: -3.6481401224598473 - - type: nauc_mrr_at_5_diff1 - value: 67.18668927352158 - - type: nauc_mrr_at_5_max - value: 41.71067825608642 - - type: nauc_mrr_at_5_std - value: -3.428397229624744 - - type: nauc_ndcg_at_1000_diff1 - value: 37.07610468655133 - - type: nauc_ndcg_at_1000_max - value: 28.224540769228234 - - type: nauc_ndcg_at_1000_std - value: 4.185248548344529 - - type: nauc_ndcg_at_100_diff1 - value: 36.42380555672608 - - type: nauc_ndcg_at_100_max - value: 27.489609981662106 - - type: nauc_ndcg_at_100_std - value: 4.054837611491843 - - type: nauc_ndcg_at_10_diff1 - value: 37.47804056929861 - - type: nauc_ndcg_at_10_max - value: 27.753495611587304 - - type: nauc_ndcg_at_10_std - value: 1.719221993401221 - - type: nauc_ndcg_at_1_diff1 - value: 69.54414023975188 - - type: nauc_ndcg_at_1_max - value: 40.65799250364921 - - type: nauc_ndcg_at_1_std - value: -5.03532335561708 - - type: nauc_ndcg_at_20_diff1 - value: 36.84564644726093 - - type: nauc_ndcg_at_20_max - value: 27.576226304248095 - - type: nauc_ndcg_at_20_std - value: 2.494939399202034 - - type: nauc_ndcg_at_3_diff1 - value: 42.2828223866538 - - type: nauc_ndcg_at_3_max - value: 30.300252457229686 - - type: nauc_ndcg_at_3_std - value: -0.8620078353970815 - - type: nauc_ndcg_at_5_diff1 - value: 39.17361365092243 - - type: nauc_ndcg_at_5_max - value: 28.90704231405507 - - type: nauc_ndcg_at_5_std - value: 0.28376558066016455 - - type: nauc_precision_at_1000_diff1 - value: -3.0942823392612357 - - type: nauc_precision_at_1000_max - value: 12.28787390319326 - - type: nauc_precision_at_1000_std - value: 25.928388012449936 - - type: nauc_precision_at_100_diff1 - value: 3.649374343629417 - - type: nauc_precision_at_100_max - value: 11.075555507772332 - - type: nauc_precision_at_100_std - value: 18.3299768701232 - - type: nauc_precision_at_10_diff1 - value: 15.415187438641182 - - type: nauc_precision_at_10_max - value: 16.52239778879397 - - type: nauc_precision_at_10_std - value: 6.837276762823966 - - type: nauc_precision_at_1_diff1 - value: 69.54414023975188 - - type: nauc_precision_at_1_max - value: 40.65799250364921 - - type: nauc_precision_at_1_std - value: -5.03532335561708 - - type: nauc_precision_at_20_diff1 - value: 11.421479372333689 - - type: nauc_precision_at_20_max - value: 14.65873574938081 - - type: nauc_precision_at_20_std - value: 9.377346877954563 - - type: nauc_precision_at_3_diff1 - value: 29.286389164119818 - - type: nauc_precision_at_3_max - value: 24.94345281552621 - - type: nauc_precision_at_3_std - value: 1.2358684556900705 - - type: nauc_precision_at_5_diff1 - value: 22.154044068448584 - - type: nauc_precision_at_5_max - value: 21.268338920126155 - - type: nauc_precision_at_5_std - value: 3.352963627463402 - - type: nauc_recall_at_1000_diff1 - value: -3.0942823392611616 - - type: nauc_recall_at_1000_max - value: 12.287873903193287 - - type: nauc_recall_at_1000_std - value: 25.92838801245011 - - type: nauc_recall_at_100_diff1 - value: 3.649374343629381 - - type: nauc_recall_at_100_max - value: 11.075555507772298 - - type: nauc_recall_at_100_std - value: 18.329976870123144 - - type: nauc_recall_at_10_diff1 - value: 15.415187438641109 - - type: nauc_recall_at_10_max - value: 16.522397788793903 - - type: nauc_recall_at_10_std - value: 6.837276762824004 - - type: nauc_recall_at_1_diff1 - value: 69.54414023975188 - - type: nauc_recall_at_1_max - value: 40.65799250364921 - - type: nauc_recall_at_1_std - value: -5.03532335561708 - - type: nauc_recall_at_20_diff1 - value: 11.42147937233372 - - type: nauc_recall_at_20_max - value: 14.65873574938076 - - type: nauc_recall_at_20_std - value: 9.377346877954627 - - type: nauc_recall_at_3_diff1 - value: 29.286389164119786 - - type: nauc_recall_at_3_max - value: 24.943452815526214 - - type: nauc_recall_at_3_std - value: 1.2358684556899975 - - type: nauc_recall_at_5_diff1 - value: 22.1540440684486 - - type: nauc_recall_at_5_max - value: 21.268338920126187 - - type: nauc_recall_at_5_std - value: 3.3529636274634282 - - type: ndcg_at_1 - value: 65.672 - - type: ndcg_at_10 - value: 56.936 - - type: ndcg_at_100 - value: 60.723000000000006 - - type: ndcg_at_1000 - value: 62.260000000000005 - - type: ndcg_at_20 - value: 58.472 - - type: ndcg_at_3 - value: 51.526 - - type: ndcg_at_5 - value: 54.37800000000001 - - type: precision_at_1 - value: 65.672 - - type: precision_at_10 - value: 12.105 - - type: precision_at_100 - value: 1.5070000000000001 - - type: precision_at_1000 - value: 0.17099999999999999 - - type: precision_at_20 - value: 6.547 - - type: precision_at_3 - value: 32.312000000000005 - - type: precision_at_5 - value: 21.648 - - type: recall_at_1 - value: 32.836 - - type: recall_at_10 - value: 60.527 - - type: recall_at_100 - value: 75.375 - - type: recall_at_1000 - value: 85.544 - - type: recall_at_20 - value: 65.469 - - type: recall_at_3 - value: 48.467 - - type: recall_at_5 - value: 54.11899999999999 - task: - type: Retrieval - - dataset: - config: default - name: MTEB ImdbClassification (default) - revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7 - split: test - type: mteb/imdb - metrics: - - type: accuracy - value: 94.90639999999999 - - type: ap - value: 92.50413995343256 - - type: ap_weighted - value: 92.50413995343256 - - type: f1 - value: 94.90353518558736 - - type: f1_weighted - value: 94.90353518558736 - - type: main_score - value: 94.90639999999999 - task: - type: Classification - - dataset: - config: default - name: MTEB InappropriatenessClassification (default) - revision: 601651fdc45ef243751676e62dd7a19f491c0285 - split: test - type: ai-forever/inappropriateness-classification - metrics: - - type: accuracy - value: 83.29023437500001 - - type: ap - value: 77.50711273019213 - - type: ap_weighted - value: 77.50711273019213 - - type: f1 - value: 82.42886423677278 - - type: f1_weighted - value: 82.42886423677278 - - type: main_score - value: 83.29023437500001 - task: - type: Classification - - dataset: - config: default - name: MTEB KinopoiskClassification (default) - revision: 5911f26666ac11af46cb9c6849d0dc80a378af24 - split: test - type: ai-forever/kinopoisk-sentiment-classification - metrics: - - type: accuracy - value: 74.88666666666667 - - type: f1 - value: 72.6314130184491 - - type: f1_weighted - value: 72.63141301844908 - - type: main_score - value: 74.88666666666667 - task: - type: Classification - - dataset: - config: ru - name: MTEB MIRACLReranking (ru) - revision: 6d1962c527217f8927fca80f890f14f36b2802af - split: dev - type: miracl/mmteb-miracl-reranking - metrics: - - type: MAP@1(MIRACL) - value: 34.256 - - type: MAP@10(MIRACL) - value: 54.418 - - type: MAP@100(MIRACL) - value: 56.224 - - type: MAP@1000(MIRACL) - value: 56.224 - - type: MAP@20(MIRACL) - value: 55.726 - - type: MAP@3(MIRACL) - value: 47.811 - - type: MAP@5(MIRACL) - value: 51.604000000000006 - - type: NDCG@1(MIRACL) - value: 55.493 - - type: NDCG@10(MIRACL) - value: 61.209 - - type: NDCG@100(MIRACL) - value: 64.768 - - type: NDCG@1000(MIRACL) - value: 64.768 - - type: NDCG@20(MIRACL) - value: 63.497 - - type: NDCG@3(MIRACL) - value: 56.259 - - type: NDCG@5(MIRACL) - value: 58.164 - - type: P@1(MIRACL) - value: 55.493 - - type: P@10(MIRACL) - value: 15.533 - - type: P@100(MIRACL) - value: 1.9529999999999998 - - type: P@1000(MIRACL) - value: 0.19499999999999998 - - type: P@20(MIRACL) - value: 8.896999999999998 - - type: P@3(MIRACL) - value: 34.509 - - type: P@5(MIRACL) - value: 25.436999999999998 - - type: Recall@1(MIRACL) - value: 34.256 - - type: Recall@10(MIRACL) - value: 69.61500000000001 - - type: Recall@100(MIRACL) - value: 79.952 - - type: Recall@1000(MIRACL) - value: 79.952 - - type: Recall@20(MIRACL) - value: 75.753 - - type: Recall@3(MIRACL) - value: 54.217999999999996 - - type: Recall@5(MIRACL) - value: 61.617999999999995 - - type: main_score - value: 61.209 - - type: nAUC_MAP@1000_diff1(MIRACL) - value: 25.375679947366475 - - type: nAUC_MAP@1000_max(MIRACL) - value: 39.17949421634556 - - type: nAUC_MAP@1000_std(MIRACL) - value: 16.668975000160252 - - type: nAUC_MAP@100_diff1(MIRACL) - value: 25.375679947366475 - - type: nAUC_MAP@100_max(MIRACL) - value: 39.17949421634556 - - type: nAUC_MAP@100_std(MIRACL) - value: 16.668975000160252 - - type: nAUC_MAP@10_diff1(MIRACL) - value: 26.475686007625093 - - type: nAUC_MAP@10_max(MIRACL) - value: 36.450804891327074 - - type: nAUC_MAP@10_std(MIRACL) - value: 15.488075690428888 - - type: nAUC_MAP@1_diff1(MIRACL) - value: 40.99538964417129 - - type: nAUC_MAP@1_max(MIRACL) - value: 19.924627548929934 - - type: nAUC_MAP@1_std(MIRACL) - value: 2.3190113335598794 - - type: nAUC_MAP@20_diff1(MIRACL) - value: 25.752468618876932 - - type: nAUC_MAP@20_max(MIRACL) - value: 38.34566370834793 - - type: nAUC_MAP@20_std(MIRACL) - value: 16.461203040560456 - - type: nAUC_MAP@3_diff1(MIRACL) - value: 30.535358312013727 - - type: nAUC_MAP@3_max(MIRACL) - value: 29.944731853212186 - - type: nAUC_MAP@3_std(MIRACL) - value: 9.103557696794429 - - type: nAUC_MAP@5_diff1(MIRACL) - value: 28.014426658247842 - - type: nAUC_MAP@5_max(MIRACL) - value: 32.85404483202943 - - type: nAUC_MAP@5_std(MIRACL) - value: 12.450517715414893 - - type: nAUC_NDCG@1000_diff1(MIRACL) - value: 18.961103796856186 - - type: nAUC_NDCG@1000_max(MIRACL) - value: 49.36987471280251 - - type: nAUC_NDCG@1000_std(MIRACL) - value: 21.734657550451335 - - type: nAUC_NDCG@100_diff1(MIRACL) - value: 18.961103796856186 - - type: nAUC_NDCG@100_max(MIRACL) - value: 49.36987471280251 - - type: nAUC_NDCG@100_std(MIRACL) - value: 21.734657550451335 - - type: nAUC_NDCG@10_diff1(MIRACL) - value: 21.983623979331 - - type: nAUC_NDCG@10_max(MIRACL) - value: 43.683160399908964 - - type: nAUC_NDCG@10_std(MIRACL) - value: 20.01372618506444 - - type: nAUC_NDCG@1_diff1(MIRACL) - value: 30.203859075266475 - - type: nAUC_NDCG@1_max(MIRACL) - value: 41.79737245030743 - - type: nAUC_NDCG@1_std(MIRACL) - value: 16.05372763120693 - - type: nAUC_NDCG@20_diff1(MIRACL) - value: 20.149932418933712 - - type: nAUC_NDCG@20_max(MIRACL) - value: 47.065303606034306 - - type: nAUC_NDCG@20_std(MIRACL) - value: 21.44815256127275 - - type: nAUC_NDCG@3_diff1(MIRACL) - value: 25.42407181388333 - - type: nAUC_NDCG@3_max(MIRACL) - value: 40.24154284953538 - - type: nAUC_NDCG@3_std(MIRACL) - value: 14.972072673678152 - - type: nAUC_NDCG@5_diff1(MIRACL) - value: 24.153034992206194 - - type: nAUC_NDCG@5_max(MIRACL) - value: 39.82178245941784 - - type: nAUC_NDCG@5_std(MIRACL) - value: 16.58411685965955 - - type: nAUC_P@1000_diff1(MIRACL) - value: -19.79420161864485 - - type: nAUC_P@1000_max(MIRACL) - value: 34.3616294815911 - - type: nAUC_P@1000_std(MIRACL) - value: 20.166807624722257 - - type: nAUC_P@100_diff1(MIRACL) - value: -19.794201618644824 - - type: nAUC_P@100_max(MIRACL) - value: 34.361629481591166 - - type: nAUC_P@100_std(MIRACL) - value: 20.166807624722274 - - type: nAUC_P@10_diff1(MIRACL) - value: -16.354560145786806 - - type: nAUC_P@10_max(MIRACL) - value: 36.05760962789994 - - type: nAUC_P@10_std(MIRACL) - value: 25.417471718298163 - - type: nAUC_P@1_diff1(MIRACL) - value: 30.203859075266475 - - type: nAUC_P@1_max(MIRACL) - value: 41.79737245030743 - - type: nAUC_P@1_std(MIRACL) - value: 16.05372763120693 - - type: nAUC_P@20_diff1(MIRACL) - value: -18.621011544449317 - - type: nAUC_P@20_max(MIRACL) - value: 35.77327581203754 - - type: nAUC_P@20_std(MIRACL) - value: 23.905223904634525 - - type: nAUC_P@3_diff1(MIRACL) - value: -6.046172734824364 - - type: nAUC_P@3_max(MIRACL) - value: 41.50549514127416 - - type: nAUC_P@3_std(MIRACL) - value: 22.16102424923467 - - type: nAUC_P@5_diff1(MIRACL) - value: -12.42960360332892 - - type: nAUC_P@5_max(MIRACL) - value: 37.02162714427216 - - type: nAUC_P@5_std(MIRACL) - value: 23.478907037340967 - - type: nAUC_Recall@1000_diff1(MIRACL) - value: -2.1015821397499126 - - type: nAUC_Recall@1000_max(MIRACL) - value: 72.4788381130068 - - type: nAUC_Recall@1000_std(MIRACL) - value: 33.13079374757504 - - type: nAUC_Recall@100_diff1(MIRACL) - value: -2.1015821397499126 - - type: nAUC_Recall@100_max(MIRACL) - value: 72.4788381130068 - - type: nAUC_Recall@100_std(MIRACL) - value: 33.13079374757504 - - type: nAUC_Recall@10_diff1(MIRACL) - value: 12.578351372233119 - - type: nAUC_Recall@10_max(MIRACL) - value: 46.853083044539666 - - type: nAUC_Recall@10_std(MIRACL) - value: 25.220743565118504 - - type: nAUC_Recall@1_diff1(MIRACL) - value: 40.99538964417129 - - type: nAUC_Recall@1_max(MIRACL) - value: 19.924627548929934 - - type: nAUC_Recall@1_std(MIRACL) - value: 2.3190113335598794 - - type: nAUC_Recall@20_diff1(MIRACL) - value: 4.959175120863266 - - type: nAUC_Recall@20_max(MIRACL) - value: 58.70067350548401 - - type: nAUC_Recall@20_std(MIRACL) - value: 30.461409638792137 - - type: nAUC_Recall@3_diff1(MIRACL) - value: 23.891732917363356 - - type: nAUC_Recall@3_max(MIRACL) - value: 31.214738377868407 - - type: nAUC_Recall@3_std(MIRACL) - value: 10.22032515759191 - - type: nAUC_Recall@5_diff1(MIRACL) - value: 18.887826092373857 - - type: nAUC_Recall@5_max(MIRACL) - value: 35.333905802137075 - - type: nAUC_Recall@5_std(MIRACL) - value: 15.818853782953793 - task: - type: Reranking - - dataset: - config: ru - name: MTEB MIRACLRetrieval (ru) - revision: main - split: dev - type: miracl/mmteb-miracl - metrics: - - type: main_score - value: 58.919 - - type: map_at_1 - value: 28.105000000000004 - - type: map_at_10 - value: 48.485 - - type: map_at_100 - value: 51.092000000000006 - - type: map_at_1000 - value: 51.197 - - type: map_at_20 - value: 49.964 - - type: map_at_3 - value: 40.702 - - type: map_at_5 - value: 44.908 - - type: mrr_at_1 - value: 53.674121405750796 - - type: mrr_at_10 - value: 65.94696105279179 - - type: mrr_at_100 - value: 66.32118832568914 - - type: mrr_at_1000 - value: 66.32790976216316 - - type: mrr_at_20 - value: 66.1976638975423 - - type: mrr_at_3 - value: 63.205537806176835 - - type: mrr_at_5 - value: 65.0745473908414 - - type: nauc_map_at_1000_diff1 - value: 28.692875448815812 - - type: nauc_map_at_1000_max - value: 12.491779369284659 - - type: nauc_map_at_1000_std - value: -2.62205081625825 - - type: nauc_map_at_100_diff1 - value: 28.645110165399174 - - type: nauc_map_at_100_max - value: 12.456098789482798 - - type: nauc_map_at_100_std - value: -2.6613261048769608 - - type: nauc_map_at_10_diff1 - value: 28.472937541575334 - - type: nauc_map_at_10_max - value: 10.810116535339406 - - type: nauc_map_at_10_std - value: -5.286588474512575 - - type: nauc_map_at_1_diff1 - value: 32.84937124647936 - - type: nauc_map_at_1_max - value: 6.422329814718059 - - type: nauc_map_at_1_std - value: -9.338607410867883 - - type: nauc_map_at_20_diff1 - value: 28.599327426899933 - - type: nauc_map_at_20_max - value: 11.82906430468277 - - type: nauc_map_at_20_std - value: -4.021436740087691 - - type: nauc_map_at_3_diff1 - value: 27.482658420810434 - - type: nauc_map_at_3_max - value: 8.017407826476449 - - type: nauc_map_at_3_std - value: -7.867867276064061 - - type: nauc_map_at_5_diff1 - value: 27.800427415149926 - - type: nauc_map_at_5_max - value: 9.039207813157995 - - type: nauc_map_at_5_std - value: -7.591204490113271 - - type: nauc_mrr_at_1000_diff1 - value: 35.78578282699573 - - type: nauc_mrr_at_1000_max - value: 20.786252125904436 - - type: nauc_mrr_at_1000_std - value: 8.583860329566223 - - type: nauc_mrr_at_100_diff1 - value: 35.775930857323694 - - type: nauc_mrr_at_100_max - value: 20.791530168252752 - - type: nauc_mrr_at_100_std - value: 8.59657930717071 - - type: nauc_mrr_at_10_diff1 - value: 35.69769923988554 - - type: nauc_mrr_at_10_max - value: 20.901225274243266 - - type: nauc_mrr_at_10_std - value: 8.708683664351218 - - type: nauc_mrr_at_1_diff1 - value: 37.68512205937764 - - type: nauc_mrr_at_1_max - value: 17.880526496865198 - - type: nauc_mrr_at_1_std - value: 4.0110867274399515 - - type: nauc_mrr_at_20_diff1 - value: 35.761942530285836 - - type: nauc_mrr_at_20_max - value: 20.834018834025102 - - type: nauc_mrr_at_20_std - value: 8.68188170294387 - - type: nauc_mrr_at_3_diff1 - value: 35.65799054328497 - - type: nauc_mrr_at_3_max - value: 20.406872858177756 - - type: nauc_mrr_at_3_std - value: 7.9695230492304105 - - type: nauc_mrr_at_5_diff1 - value: 36.00029293304689 - - type: nauc_mrr_at_5_max - value: 20.893681203274397 - - type: nauc_mrr_at_5_std - value: 8.608032961961529 - - type: nauc_ndcg_at_1000_diff1 - value: 30.283370916340395 - - type: nauc_ndcg_at_1000_max - value: 17.45515207920199 - - type: nauc_ndcg_at_1000_std - value: 5.500817122521028 - - type: nauc_ndcg_at_100_diff1 - value: 29.37087750574036 - - type: nauc_ndcg_at_100_max - value: 17.1659856137934 - - type: nauc_ndcg_at_100_std - value: 5.740672030272012 - - type: nauc_ndcg_at_10_diff1 - value: 29.190898853994153 - - type: nauc_ndcg_at_10_max - value: 13.581733885756453 - - type: nauc_ndcg_at_10_std - value: -1.1925279328006109 - - type: nauc_ndcg_at_1_diff1 - value: 37.68512205937764 - - type: nauc_ndcg_at_1_max - value: 17.880526496865198 - - type: nauc_ndcg_at_1_std - value: 4.0110867274399515 - - type: nauc_ndcg_at_20_diff1 - value: 29.37764766181379 - - type: nauc_ndcg_at_20_max - value: 15.345088185381453 - - type: nauc_ndcg_at_20_std - value: 1.5388051841648351 - - type: nauc_ndcg_at_3_diff1 - value: 28.550969354690576 - - type: nauc_ndcg_at_3_max - value: 13.205099776372926 - - type: nauc_ndcg_at_3_std - value: 0.12608790067765707 - - type: nauc_ndcg_at_5_diff1 - value: 29.262808940095304 - - type: nauc_ndcg_at_5_max - value: 12.34844355941903 - - type: nauc_ndcg_at_5_std - value: -2.503299305763401 - - type: nauc_precision_at_1000_diff1 - value: -4.946929314275426 - - type: nauc_precision_at_1000_max - value: 14.318369622332805 - - type: nauc_precision_at_1000_std - value: 29.953649623967998 - - type: nauc_precision_at_100_diff1 - value: -4.8754407088382665 - - type: nauc_precision_at_100_max - value: 16.437239890966012 - - type: nauc_precision_at_100_std - value: 32.86582107306089 - - type: nauc_precision_at_10_diff1 - value: 3.50201515428894 - - type: nauc_precision_at_10_max - value: 17.75684080894953 - - type: nauc_precision_at_10_std - value: 22.110419776166054 - - type: nauc_precision_at_1_diff1 - value: 37.68512205937764 - - type: nauc_precision_at_1_max - value: 17.880526496865198 - - type: nauc_precision_at_1_std - value: 4.0110867274399515 - - type: nauc_precision_at_20_diff1 - value: 0.3624861246639223 - - type: nauc_precision_at_20_max - value: 18.774584939318544 - - type: nauc_precision_at_20_std - value: 27.061134898065724 - - type: nauc_precision_at_3_diff1 - value: 12.364445300291129 - - type: nauc_precision_at_3_max - value: 17.198756066552694 - - type: nauc_precision_at_3_std - value: 13.396863299435207 - - type: nauc_precision_at_5_diff1 - value: 7.903105219850088 - - type: nauc_precision_at_5_max - value: 17.20630661570393 - - type: nauc_precision_at_5_std - value: 16.41195105173974 - - type: nauc_recall_at_1000_diff1 - value: 38.98954496312062 - - type: nauc_recall_at_1000_max - value: 55.33728064769203 - - type: nauc_recall_at_1000_std - value: 63.0550070852272 - - type: nauc_recall_at_100_diff1 - value: 12.772525292034661 - - type: nauc_recall_at_100_max - value: 21.51166499991489 - - type: nauc_recall_at_100_std - value: 28.55416139778685 - - type: nauc_recall_at_10_diff1 - value: 21.39711887049693 - - type: nauc_recall_at_10_max - value: 7.632620412302538 - - type: nauc_recall_at_10_std - value: -5.738894810245653 - - type: nauc_recall_at_1_diff1 - value: 32.84937124647936 - - type: nauc_recall_at_1_max - value: 6.422329814718059 - - type: nauc_recall_at_1_std - value: -9.338607410867883 - - type: nauc_recall_at_20_diff1 - value: 19.502187424365978 - - type: nauc_recall_at_20_max - value: 10.660163096971175 - - type: nauc_recall_at_20_std - value: 0.9197930988253975 - - type: nauc_recall_at_3_diff1 - value: 21.4039473712599 - - type: nauc_recall_at_3_max - value: 4.9820152592443065 - - type: nauc_recall_at_3_std - value: -8.873734870059172 - - type: nauc_recall_at_5_diff1 - value: 21.487915917217766 - - type: nauc_recall_at_5_max - value: 5.118393026118493 - - type: nauc_recall_at_5_std - value: -10.131702930834898 - - type: ndcg_at_1 - value: 53.674 - - type: ndcg_at_10 - value: 58.919 - - type: ndcg_at_100 - value: 65.736 - - type: ndcg_at_1000 - value: 66.932 - - type: ndcg_at_20 - value: 61.97200000000001 - - type: ndcg_at_3 - value: 52.019999999999996 - - type: ndcg_at_5 - value: 54.63099999999999 - - type: precision_at_1 - value: 53.674 - - type: precision_at_10 - value: 17.332 - - type: precision_at_100 - value: 2.546 - - type: precision_at_1000 - value: 0.27899999999999997 - - type: precision_at_20 - value: 10.152 - - type: precision_at_3 - value: 34.798 - - type: precision_at_5 - value: 26.805 - - type: recall_at_1 - value: 28.105000000000004 - - type: recall_at_10 - value: 70.024 - - type: recall_at_100 - value: 91.739 - - type: recall_at_1000 - value: 98.131 - - type: recall_at_20 - value: 78.385 - - type: recall_at_3 - value: 48.199 - - type: recall_at_5 - value: 58.495 - task: - type: Retrieval - - dataset: - config: default - name: MTEB MSMARCO (default) - revision: c5a29a104738b98a9e76336939199e264163d4a0 - split: test - type: mteb/msmarco - metrics: - - type: main_score - value: 61.953 - - type: map_at_1 - value: 2.2950000000000004 - - type: map_at_10 - value: 13.425 - - type: map_at_100 - value: 36.994 - - type: map_at_1000 - value: 45.477000000000004 - - type: map_at_20 - value: 20.541999999999998 - - type: map_at_3 - value: 5.632000000000001 - - type: map_at_5 - value: 7.9350000000000005 - - type: mrr_at_1 - value: 86.04651162790698 - - type: mrr_at_10 - value: 90.968992248062 - - type: mrr_at_100 - value: 90.968992248062 - - type: mrr_at_1000 - value: 90.968992248062 - - type: mrr_at_20 - value: 90.968992248062 - - type: mrr_at_3 - value: 89.92248062015503 - - type: mrr_at_5 - value: 90.968992248062 - - type: nauc_map_at_1000_diff1 - value: -30.11984354598437 - - type: nauc_map_at_1000_max - value: 41.121622214784544 - - type: nauc_map_at_1000_std - value: 53.94915916714401 - - type: nauc_map_at_100_diff1 - value: -18.474186072111014 - - type: nauc_map_at_100_max - value: 32.55771055611539 - - type: nauc_map_at_100_std - value: 34.850257861392365 - - type: nauc_map_at_10_diff1 - value: 12.626067523953438 - - type: nauc_map_at_10_max - value: 13.292172274891447 - - type: nauc_map_at_10_std - value: -2.7683801353352253 - - type: nauc_map_at_1_diff1 - value: 44.2949717144564 - - type: nauc_map_at_1_max - value: -5.304125695825888 - - type: nauc_map_at_1_std - value: -29.85748050243832 - - type: nauc_map_at_20_diff1 - value: 1.6976308361799894 - - type: nauc_map_at_20_max - value: 18.87963530770625 - - type: nauc_map_at_20_std - value: 6.0785987235463255 - - type: nauc_map_at_3_diff1 - value: 36.09132430429921 - - type: nauc_map_at_3_max - value: 2.8220999625196037 - - type: nauc_map_at_3_std - value: -21.476895853466463 - - type: nauc_map_at_5_diff1 - value: 27.5419170591421 - - type: nauc_map_at_5_max - value: 9.290150393583366 - - type: nauc_map_at_5_std - value: -13.742406800486986 - - type: nauc_mrr_at_1000_diff1 - value: -7.6099403523558 - - type: nauc_mrr_at_1000_max - value: 75.01353061667479 - - type: nauc_mrr_at_1000_std - value: 68.72722608823327 - - type: nauc_mrr_at_100_diff1 - value: -7.6099403523558 - - type: nauc_mrr_at_100_max - value: 75.01353061667479 - - type: nauc_mrr_at_100_std - value: 68.72722608823327 - - type: nauc_mrr_at_10_diff1 - value: -7.6099403523558 - - type: nauc_mrr_at_10_max - value: 75.01353061667479 - - type: nauc_mrr_at_10_std - value: 68.72722608823327 - - type: nauc_mrr_at_1_diff1 - value: -7.2278364840418385 - - type: nauc_mrr_at_1_max - value: 70.07087438233302 - - type: nauc_mrr_at_1_std - value: 62.28193431865652 - - type: nauc_mrr_at_20_diff1 - value: -7.6099403523558 - - type: nauc_mrr_at_20_max - value: 75.01353061667479 - - type: nauc_mrr_at_20_std - value: 68.72722608823327 - - type: nauc_mrr_at_3_diff1 - value: -2.664317491139411 - - type: nauc_mrr_at_3_max - value: 76.91690813390306 - - type: nauc_mrr_at_3_std - value: 70.41867051808181 - - type: nauc_mrr_at_5_diff1 - value: -7.6099403523558 - - type: nauc_mrr_at_5_max - value: 75.01353061667479 - - type: nauc_mrr_at_5_std - value: 68.72722608823327 - - type: nauc_ndcg_at_1000_diff1 - value: -34.58820972084936 - - type: nauc_ndcg_at_1000_max - value: 54.01529717339395 - - type: nauc_ndcg_at_1000_std - value: 59.316374712522645 - - type: nauc_ndcg_at_100_diff1 - value: -26.640288133521523 - - type: nauc_ndcg_at_100_max - value: 35.13407553215567 - - type: nauc_ndcg_at_100_std - value: 47.90610288563949 - - type: nauc_ndcg_at_10_diff1 - value: -30.16559486015698 - - type: nauc_ndcg_at_10_max - value: 38.910360318642525 - - type: nauc_ndcg_at_10_std - value: 42.68661569167478 - - type: nauc_ndcg_at_1_diff1 - value: -1.5593104420333765 - - type: nauc_ndcg_at_1_max - value: 42.61039841149629 - - type: nauc_ndcg_at_1_std - value: 35.840800888923035 - - type: nauc_ndcg_at_20_diff1 - value: -30.77391764501866 - - type: nauc_ndcg_at_20_max - value: 34.50283605663452 - - type: nauc_ndcg_at_20_std - value: 37.59483221610664 - - type: nauc_ndcg_at_3_diff1 - value: -15.520713715926346 - - type: nauc_ndcg_at_3_max - value: 45.78611982790275 - - type: nauc_ndcg_at_3_std - value: 45.97839281793152 - - type: nauc_ndcg_at_5_diff1 - value: -23.85285278088875 - - type: nauc_ndcg_at_5_max - value: 44.39780563283533 - - type: nauc_ndcg_at_5_std - value: 42.39949056653093 - - type: nauc_precision_at_1000_diff1 - value: -30.417558924768134 - - type: nauc_precision_at_1000_max - value: 13.548159537058153 - - type: nauc_precision_at_1000_std - value: 37.25751589837348 - - type: nauc_precision_at_100_diff1 - value: -36.46125649080512 - - type: nauc_precision_at_100_max - value: 20.74186636114751 - - type: nauc_precision_at_100_std - value: 47.312413368139154 - - type: nauc_precision_at_10_diff1 - value: -59.04379077622773 - - type: nauc_precision_at_10_max - value: 46.66289912456372 - - type: nauc_precision_at_10_std - value: 62.769623439608665 - - type: nauc_precision_at_1_diff1 - value: -7.2278364840418385 - - type: nauc_precision_at_1_max - value: 70.07087438233302 - - type: nauc_precision_at_1_std - value: 62.28193431865652 - - type: nauc_precision_at_20_diff1 - value: -53.10996336661844 - - type: nauc_precision_at_20_max - value: 38.09482517069248 - - type: nauc_precision_at_20_std - value: 56.736115781817766 - - type: nauc_precision_at_3_diff1 - value: -36.999469625002206 - - type: nauc_precision_at_3_max - value: 64.12921150590219 - - type: nauc_precision_at_3_std - value: 64.78671082210093 - - type: nauc_precision_at_5_diff1 - value: -46.500845210255335 - - type: nauc_precision_at_5_max - value: 59.4191275770431 - - type: nauc_precision_at_5_std - value: 68.00855518751847 - - type: nauc_recall_at_1000_diff1 - value: -24.555235403948775 - - type: nauc_recall_at_1000_max - value: 54.22435135708867 - - type: nauc_recall_at_1000_std - value: 49.31140174115312 - - type: nauc_recall_at_100_diff1 - value: -3.7001471510828496 - - type: nauc_recall_at_100_max - value: 19.21098260931646 - - type: nauc_recall_at_100_std - value: 18.131361156635535 - - type: nauc_recall_at_10_diff1 - value: 16.09333751982658 - - type: nauc_recall_at_10_max - value: 7.004596323208447 - - type: nauc_recall_at_10_std - value: -9.042361327667667 - - type: nauc_recall_at_1_diff1 - value: 44.2949717144564 - - type: nauc_recall_at_1_max - value: -5.304125695825888 - - type: nauc_recall_at_1_std - value: -29.85748050243832 - - type: nauc_recall_at_20_diff1 - value: 9.048625900019916 - - type: nauc_recall_at_20_max - value: 10.488512958255308 - - type: nauc_recall_at_20_std - value: -3.079039962366688 - - type: nauc_recall_at_3_diff1 - value: 36.3729568590647 - - type: nauc_recall_at_3_max - value: 1.6785799970091961 - - type: nauc_recall_at_3_std - value: -22.297408897380535 - - type: nauc_recall_at_5_diff1 - value: 27.404925186321762 - - type: nauc_recall_at_5_max - value: 4.594462203289419 - - type: nauc_recall_at_5_std - value: -17.585190330631452 - - type: ndcg_at_1 - value: 66.279 - - type: ndcg_at_10 - value: 61.953 - - type: ndcg_at_100 - value: 61.38400000000001 - - type: ndcg_at_1000 - value: 71.18299999999999 - - type: ndcg_at_20 - value: 61.391 - - type: ndcg_at_3 - value: 63.244 - - type: ndcg_at_5 - value: 61.757 - - type: precision_at_1 - value: 86.047 - - type: precision_at_10 - value: 73.256 - - type: precision_at_100 - value: 37.535000000000004 - - type: precision_at_1000 - value: 7.579 - - type: precision_at_20 - value: 65.349 - - type: precision_at_3 - value: 79.845 - - type: precision_at_5 - value: 75.81400000000001 - - type: recall_at_1 - value: 2.2950000000000004 - - type: recall_at_10 - value: 15.101 - - type: recall_at_100 - value: 51.615 - - type: recall_at_1000 - value: 82.471 - - type: recall_at_20 - value: 24.096 - - type: recall_at_3 - value: 5.773 - - type: recall_at_5 - value: 8.5 - task: - type: Retrieval - - dataset: - config: en - name: MTEB MTOPDomainClassification (en) - revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf - split: test - type: mteb/mtop_domain - metrics: - - type: accuracy - value: 95.35111719106247 - - type: f1 - value: 95.18886239707825 - - type: f1_weighted - value: 95.35614308126868 - - type: main_score - value: 95.35111719106247 - task: - type: Classification - - dataset: - config: en - name: MTEB MTOPIntentClassification (en) - revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba - split: test - type: mteb/mtop_intent - metrics: - - type: accuracy - value: 80.47423620611035 - - type: f1 - value: 58.99139430808884 - - type: f1_weighted - value: 82.26823539179561 - - type: main_score - value: 80.47423620611035 - task: - type: Classification - - dataset: - config: ru - name: MTEB MassiveIntentClassification (ru) - revision: 4672e20407010da34463acc759c162ca9734bca6 - split: test - type: mteb/amazon_massive_intent - metrics: - - type: accuracy - value: 73.44317417619368 - - type: f1 - value: 70.41959602535415 - - type: f1_weighted - value: 72.46446281404538 - - type: main_score - value: 73.44317417619368 - task: - type: Classification - - dataset: - config: ru - name: MTEB MassiveIntentClassification (ru) - revision: 4672e20407010da34463acc759c162ca9734bca6 - split: validation - type: mteb/amazon_massive_intent - metrics: - - type: accuracy - value: 72.94146581406788 - - type: f1 - value: 68.55171702555123 - - type: f1_weighted - value: 71.8914242086976 - - type: main_score - value: 72.94146581406788 - task: - type: Classification - - dataset: - config: ru - name: MTEB MassiveScenarioClassification (ru) - revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 - split: test - type: mteb/amazon_massive_scenario - metrics: - - type: accuracy - value: 78.17417619367855 - - type: f1 - value: 77.00893008191852 - - type: f1_weighted - value: 77.94617575337399 - - type: main_score - value: 78.17417619367855 - task: - type: Classification - - dataset: - config: ru - name: MTEB MassiveScenarioClassification (ru) - revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 - split: validation - type: mteb/amazon_massive_scenario - metrics: - - type: accuracy - value: 77.8160354156419 - - type: f1 - value: 76.67169331969356 - - type: f1_weighted - value: 77.54451044923486 - - type: main_score - value: 77.8160354156419 - task: - type: Classification - - dataset: - config: default - name: MTEB NFCorpus (default) - revision: ec0fa4fe99da2ff19ca1214b7966684033a58814 - split: test - type: mteb/nfcorpus - metrics: - - type: main_score - value: 36.028999999999996 - - type: map_at_1 - value: 5.9319999999999995 - - type: map_at_10 - value: 13.353000000000002 - - type: map_at_100 - value: 17.0 - - type: map_at_1000 - value: 18.565 - - type: map_at_20 - value: 14.802999999999999 - - type: map_at_3 - value: 9.83 - - type: map_at_5 - value: 11.472 - - type: mrr_at_1 - value: 46.749226006191954 - - type: mrr_at_10 - value: 56.79627991547497 - - type: mrr_at_100 - value: 57.37706490767912 - - type: mrr_at_1000 - value: 57.41967573448375 - - type: mrr_at_20 - value: 57.1211287532972 - - type: mrr_at_3 - value: 54.95356037151704 - - type: mrr_at_5 - value: 56.06811145510837 - - type: nauc_map_at_1000_diff1 - value: 35.301138973780674 - - type: nauc_map_at_1000_max - value: 34.42496916780575 - - type: nauc_map_at_1000_std - value: 20.929139100817633 - - type: nauc_map_at_100_diff1 - value: 36.11891724859925 - - type: nauc_map_at_100_max - value: 34.42849138137279 - - type: nauc_map_at_100_std - value: 19.11350296198399 - - type: nauc_map_at_10_diff1 - value: 41.049004397656205 - - type: nauc_map_at_10_max - value: 31.02146168233007 - - type: nauc_map_at_10_std - value: 9.498859153190814 - - type: nauc_map_at_1_diff1 - value: 51.1222877824682 - - type: nauc_map_at_1_max - value: 18.489074115470764 - - type: nauc_map_at_1_std - value: -2.614775958130393 - - type: nauc_map_at_20_diff1 - value: 38.199531357069134 - - type: nauc_map_at_20_max - value: 33.1062790373509 - - type: nauc_map_at_20_std - value: 13.927078976412968 - - type: nauc_map_at_3_diff1 - value: 47.42097675315478 - - type: nauc_map_at_3_max - value: 26.5657921989551 - - type: nauc_map_at_3_std - value: 4.189135527020967 - - type: nauc_map_at_5_diff1 - value: 44.71175314512709 - - type: nauc_map_at_5_max - value: 28.883088132977917 - - type: nauc_map_at_5_std - value: 5.582422026464234 - - type: nauc_mrr_at_1000_diff1 - value: 33.71354781002677 - - type: nauc_mrr_at_1000_max - value: 43.72641769159078 - - type: nauc_mrr_at_1000_std - value: 27.54102562853434 - - type: nauc_mrr_at_100_diff1 - value: 33.69597757409847 - - type: nauc_mrr_at_100_max - value: 43.74809406750261 - - type: nauc_mrr_at_100_std - value: 27.584428643679402 - - type: nauc_mrr_at_10_diff1 - value: 33.8314981272887 - - type: nauc_mrr_at_10_max - value: 43.60703620245724 - - type: nauc_mrr_at_10_std - value: 27.24838397886887 - - type: nauc_mrr_at_1_diff1 - value: 31.34357704107609 - - type: nauc_mrr_at_1_max - value: 39.593265875570275 - - type: nauc_mrr_at_1_std - value: 21.334421193946483 - - type: nauc_mrr_at_20_diff1 - value: 33.76655353328545 - - type: nauc_mrr_at_20_max - value: 43.60548690804639 - - type: nauc_mrr_at_20_std - value: 27.277618485201486 - - type: nauc_mrr_at_3_diff1 - value: 34.498566125110585 - - type: nauc_mrr_at_3_max - value: 43.051850338353184 - - type: nauc_mrr_at_3_std - value: 26.587786329119016 - - type: nauc_mrr_at_5_diff1 - value: 34.28107082592715 - - type: nauc_mrr_at_5_max - value: 43.77759543090162 - - type: nauc_mrr_at_5_std - value: 27.51020660226881 - - type: nauc_ndcg_at_1000_diff1 - value: 34.02416868205839 - - type: nauc_ndcg_at_1000_max - value: 45.616471969818406 - - type: nauc_ndcg_at_1000_std - value: 35.73939273646117 - - type: nauc_ndcg_at_100_diff1 - value: 31.751037415300797 - - type: nauc_ndcg_at_100_max - value: 39.95194747369657 - - type: nauc_ndcg_at_100_std - value: 29.099636537256384 - - type: nauc_ndcg_at_10_diff1 - value: 25.878621628641923 - - type: nauc_ndcg_at_10_max - value: 37.49225635027015 - - type: nauc_ndcg_at_10_std - value: 25.29534976285755 - - type: nauc_ndcg_at_1_diff1 - value: 31.625392306396943 - - type: nauc_ndcg_at_1_max - value: 39.04410623112782 - - type: nauc_ndcg_at_1_std - value: 22.316304086718933 - - type: nauc_ndcg_at_20_diff1 - value: 27.06245182876024 - - type: nauc_ndcg_at_20_max - value: 36.99168550915193 - - type: nauc_ndcg_at_20_std - value: 26.15811674137553 - - type: nauc_ndcg_at_3_diff1 - value: 26.07107932833021 - - type: nauc_ndcg_at_3_max - value: 38.04734587855705 - - type: nauc_ndcg_at_3_std - value: 22.940589033430346 - - type: nauc_ndcg_at_5_diff1 - value: 25.97420963086261 - - type: nauc_ndcg_at_5_max - value: 39.0151165636691 - - type: nauc_ndcg_at_5_std - value: 24.72377772080157 - - type: nauc_precision_at_1000_diff1 - value: -11.02371885050686 - - type: nauc_precision_at_1000_max - value: -5.068154375781828 - - type: nauc_precision_at_1000_std - value: 11.85905313652616 - - type: nauc_precision_at_100_diff1 - value: -8.27923233471529 - - type: nauc_precision_at_100_max - value: 9.303415360666891 - - type: nauc_precision_at_100_std - value: 26.935679867455153 - - type: nauc_precision_at_10_diff1 - value: 3.558842040982842 - - type: nauc_precision_at_10_max - value: 30.62292167383454 - - type: nauc_precision_at_10_std - value: 30.831543521961724 - - type: nauc_precision_at_1_diff1 - value: 32.18214923067921 - - type: nauc_precision_at_1_max - value: 40.03416082124917 - - type: nauc_precision_at_1_std - value: 21.941362014075718 - - type: nauc_precision_at_20_diff1 - value: -2.216008179688629 - - type: nauc_precision_at_20_max - value: 23.696772836984596 - - type: nauc_precision_at_20_std - value: 32.32501578135179 - - type: nauc_precision_at_3_diff1 - value: 17.327428569744058 - - type: nauc_precision_at_3_max - value: 36.966310259945814 - - type: nauc_precision_at_3_std - value: 25.40310439934565 - - type: nauc_precision_at_5_diff1 - value: 11.325073410234284 - - type: nauc_precision_at_5_max - value: 35.27087123202698 - - type: nauc_precision_at_5_std - value: 27.43381915118116 - - type: nauc_recall_at_1000_diff1 - value: 19.343460354273635 - - type: nauc_recall_at_1000_max - value: 21.188913677267575 - - type: nauc_recall_at_1000_std - value: 18.75561707338635 - - type: nauc_recall_at_100_diff1 - value: 23.542278286597558 - - type: nauc_recall_at_100_max - value: 25.795529811599554 - - type: nauc_recall_at_100_std - value: 19.20436682986688 - - type: nauc_recall_at_10_diff1 - value: 36.57350490861877 - - type: nauc_recall_at_10_max - value: 26.98041459230936 - - type: nauc_recall_at_10_std - value: 6.448693265410069 - - type: nauc_recall_at_1_diff1 - value: 51.1222877824682 - - type: nauc_recall_at_1_max - value: 18.489074115470764 - - type: nauc_recall_at_1_std - value: -2.614775958130393 - - type: nauc_recall_at_20_diff1 - value: 31.934925086533646 - - type: nauc_recall_at_20_max - value: 28.186182157743715 - - type: nauc_recall_at_20_std - value: 10.990757869684472 - - type: nauc_recall_at_3_diff1 - value: 47.31528648142464 - - type: nauc_recall_at_3_max - value: 26.36699566948872 - - type: nauc_recall_at_3_std - value: 4.162483410134025 - - type: nauc_recall_at_5_diff1 - value: 41.52364816245154 - - type: nauc_recall_at_5_max - value: 27.757237222978056 - - type: nauc_recall_at_5_std - value: 4.477114493542167 - - type: ndcg_at_1 - value: 44.891999999999996 - - type: ndcg_at_10 - value: 36.028999999999996 - - type: ndcg_at_100 - value: 33.339999999999996 - - type: ndcg_at_1000 - value: 42.584 - - type: ndcg_at_20 - value: 33.653 - - type: ndcg_at_3 - value: 41.747 - - type: ndcg_at_5 - value: 39.204 - - type: precision_at_1 - value: 46.44 - - type: precision_at_10 - value: 26.625 - - type: precision_at_100 - value: 8.44 - - type: precision_at_1000 - value: 2.168 - - type: precision_at_20 - value: 19.737 - - type: precision_at_3 - value: 39.732 - - type: precision_at_5 - value: 33.994 - - type: recall_at_1 - value: 5.9319999999999995 - - type: recall_at_10 - value: 17.523 - - type: recall_at_100 - value: 34.213 - - type: recall_at_1000 - value: 67.244 - - type: recall_at_20 - value: 21.367 - - type: recall_at_3 - value: 10.953 - - type: recall_at_5 - value: 13.675999999999998 - task: - type: Retrieval - - dataset: - config: default - name: MTEB NQ (default) - revision: b774495ed302d8c44a3a7ea25c90dbce03968f31 - split: test - type: mteb/nq - metrics: - - type: main_score - value: 47.417 - - type: map_at_1 - value: 25.089 - - type: map_at_10 - value: 39.582 - - type: map_at_100 - value: 40.891 - - type: map_at_1000 - value: 40.93 - - type: map_at_20 - value: 40.451 - - type: map_at_3 - value: 34.998000000000005 - - type: map_at_5 - value: 37.604 - - type: mrr_at_1 - value: 28.418308227114714 - - type: mrr_at_10 - value: 42.05473247622723 - - type: mrr_at_100 - value: 43.080018588766386 - - type: mrr_at_1000 - value: 43.10671917944525 - - type: mrr_at_20 - value: 42.75932308461006 - - type: mrr_at_3 - value: 38.15662417921973 - - type: mrr_at_5 - value: 40.42487446890695 - - type: nauc_map_at_1000_diff1 - value: 28.91528249267823 - - type: nauc_map_at_1000_max - value: 12.793914414537005 - - type: nauc_map_at_1000_std - value: -1.092950755496448 - - type: nauc_map_at_100_diff1 - value: 28.91080883187314 - - type: nauc_map_at_100_max - value: 12.819170484926945 - - type: nauc_map_at_100_std - value: -1.0715029364921733 - - type: nauc_map_at_10_diff1 - value: 28.78480452256528 - - type: nauc_map_at_10_max - value: 12.576414045375905 - - type: nauc_map_at_10_std - value: -1.5015042135306234 - - type: nauc_map_at_1_diff1 - value: 33.03005074446969 - - type: nauc_map_at_1_max - value: 8.59468158008318 - - type: nauc_map_at_1_std - value: -4.299975654790197 - - type: nauc_map_at_20_diff1 - value: 28.914182108026242 - - type: nauc_map_at_20_max - value: 12.809670683056721 - - type: nauc_map_at_20_std - value: -1.1939469617180538 - - type: nauc_map_at_3_diff1 - value: 28.991050451790972 - - type: nauc_map_at_3_max - value: 11.180438925592629 - - type: nauc_map_at_3_std - value: -3.314676077526647 - - type: nauc_map_at_5_diff1 - value: 28.758243799705514 - - type: nauc_map_at_5_max - value: 12.0624973622888 - - type: nauc_map_at_5_std - value: -2.2694243325484504 - - type: nauc_mrr_at_1000_diff1 - value: 28.27168423988733 - - type: nauc_mrr_at_1000_max - value: 13.928637140455786 - - type: nauc_mrr_at_1000_std - value: 1.2672195165656992 - - type: nauc_mrr_at_100_diff1 - value: 28.270359379156183 - - type: nauc_mrr_at_100_max - value: 13.951816084768742 - - type: nauc_mrr_at_100_std - value: 1.2938704185608485 - - type: nauc_mrr_at_10_diff1 - value: 28.003472749770904 - - type: nauc_mrr_at_10_max - value: 13.931095921919853 - - type: nauc_mrr_at_10_std - value: 1.1269493369630486 - - type: nauc_mrr_at_1_diff1 - value: 32.44528183913001 - - type: nauc_mrr_at_1_max - value: 10.425874491501148 - - type: nauc_mrr_at_1_std - value: -1.223877452463658 - - type: nauc_mrr_at_20_diff1 - value: 28.238168052236727 - - type: nauc_mrr_at_20_max - value: 14.021546198308194 - - type: nauc_mrr_at_20_std - value: 1.3081243309456347 - - type: nauc_mrr_at_3_diff1 - value: 28.067272464887296 - - type: nauc_mrr_at_3_max - value: 12.695913784346569 - - type: nauc_mrr_at_3_std - value: -0.18550418554088222 - - type: nauc_mrr_at_5_diff1 - value: 27.745708979354827 - - type: nauc_mrr_at_5_max - value: 13.699813791475782 - - type: nauc_mrr_at_5_std - value: 0.8500026849272696 - - type: nauc_ndcg_at_1000_diff1 - value: 28.24687682394225 - - type: nauc_ndcg_at_1000_max - value: 14.847726899183506 - - type: nauc_ndcg_at_1000_std - value: 1.6167349963007875 - - type: nauc_ndcg_at_100_diff1 - value: 28.14488488771258 - - type: nauc_ndcg_at_100_max - value: 15.618743122645244 - - type: nauc_ndcg_at_100_std - value: 2.45572131120658 - - type: nauc_ndcg_at_10_diff1 - value: 27.441464404432956 - - type: nauc_ndcg_at_10_max - value: 14.8506532012367 - - type: nauc_ndcg_at_10_std - value: 0.8782842057690032 - - type: nauc_ndcg_at_1_diff1 - value: 32.54713711215239 - - type: nauc_ndcg_at_1_max - value: 10.478653863508299 - - type: nauc_ndcg_at_1_std - value: -1.1496114859411906 - - type: nauc_ndcg_at_20_diff1 - value: 28.045210967473693 - - type: nauc_ndcg_at_20_max - value: 15.644714013409134 - - type: nauc_ndcg_at_20_std - value: 1.9062527641824571 - - type: nauc_ndcg_at_3_diff1 - value: 27.668700218726016 - - type: nauc_ndcg_at_3_max - value: 12.209702851489775 - - type: nauc_ndcg_at_3_std - value: -2.322334570511838 - - type: nauc_ndcg_at_5_diff1 - value: 27.16752269329571 - - type: nauc_ndcg_at_5_max - value: 13.840657657672923 - - type: nauc_ndcg_at_5_std - value: -0.5167877238881974 - - type: nauc_precision_at_1000_diff1 - value: -6.0278343340760685 - - type: nauc_precision_at_1000_max - value: 11.225657752379693 - - type: nauc_precision_at_1000_std - value: 19.122936171005318 - - type: nauc_precision_at_100_diff1 - value: -0.3778227983882008 - - type: nauc_precision_at_100_max - value: 18.72448662528021 - - type: nauc_precision_at_100_std - value: 23.65966502022462 - - type: nauc_precision_at_10_diff1 - value: 12.958694637851698 - - type: nauc_precision_at_10_max - value: 19.3564028554887 - - type: nauc_precision_at_10_std - value: 12.5127450303591 - - type: nauc_precision_at_1_diff1 - value: 32.54713711215239 - - type: nauc_precision_at_1_max - value: 10.478653863508299 - - type: nauc_precision_at_1_std - value: -1.1496114859411906 - - type: nauc_precision_at_20_diff1 - value: 9.14957988107668 - - type: nauc_precision_at_20_max - value: 21.2077388823531 - - type: nauc_precision_at_20_std - value: 18.32152485970381 - - type: nauc_precision_at_3_diff1 - value: 21.0372683571967 - - type: nauc_precision_at_3_max - value: 15.684100593309513 - - type: nauc_precision_at_3_std - value: 2.4580907891125503 - - type: nauc_precision_at_5_diff1 - value: 17.20617330695405 - - type: nauc_precision_at_5_max - value: 18.211885404364732 - - type: nauc_precision_at_5_std - value: 7.436932427752223 - - type: nauc_recall_at_1000_diff1 - value: 40.28580404519019 - - type: nauc_recall_at_1000_max - value: 60.50252714882125 - - type: nauc_recall_at_1000_std - value: 65.26984741399421 - - type: nauc_recall_at_100_diff1 - value: 26.517111029668662 - - type: nauc_recall_at_100_max - value: 43.37392004137636 - - type: nauc_recall_at_100_std - value: 36.39302655748604 - - type: nauc_recall_at_10_diff1 - value: 22.85248270157389 - - type: nauc_recall_at_10_max - value: 20.595760178488483 - - type: nauc_recall_at_10_std - value: 6.07222355370645 - - type: nauc_recall_at_1_diff1 - value: 33.03005074446969 - - type: nauc_recall_at_1_max - value: 8.59468158008318 - - type: nauc_recall_at_1_std - value: -4.299975654790197 - - type: nauc_recall_at_20_diff1 - value: 25.249292416064677 - - type: nauc_recall_at_20_max - value: 27.01732868361449 - - type: nauc_recall_at_20_std - value: 13.03723631965964 - - type: nauc_recall_at_3_diff1 - value: 24.07995496056652 - - type: nauc_recall_at_3_max - value: 13.160951322480182 - - type: nauc_recall_at_3_std - value: -2.4282680235922993 - - type: nauc_recall_at_5_diff1 - value: 22.48576829416849 - - type: nauc_recall_at_5_max - value: 16.82595398561844 - - type: nauc_recall_at_5_std - value: 1.5133407674177162 - - type: ndcg_at_1 - value: 28.388999999999996 - - type: ndcg_at_10 - value: 47.417 - - type: ndcg_at_100 - value: 52.903999999999996 - - type: ndcg_at_1000 - value: 53.754000000000005 - - type: ndcg_at_20 - value: 50.282000000000004 - - type: ndcg_at_3 - value: 38.682 - - type: ndcg_at_5 - value: 43.059 - - type: precision_at_1 - value: 28.388999999999996 - - type: precision_at_10 - value: 8.239 - - type: precision_at_100 - value: 1.124 - - type: precision_at_1000 - value: 0.12 - - type: precision_at_20 - value: 4.7829999999999995 - - type: precision_at_3 - value: 17.922 - - type: precision_at_5 - value: 13.285 - - type: recall_at_1 - value: 25.089 - - type: recall_at_10 - value: 68.931 - - type: recall_at_100 - value: 92.54299999999999 - - type: recall_at_1000 - value: 98.774 - - type: recall_at_20 - value: 79.633 - - type: recall_at_3 - value: 46.193 - - type: recall_at_5 - value: 56.257000000000005 - task: - type: Retrieval - - dataset: - config: default - name: MTEB QuoraRetrieval (default) - revision: e4e08e0b7dbe3c8700f0daef558ff32256715259 - split: test - type: mteb/quora - metrics: - - type: main_score - value: 89.237 - - type: map_at_1 - value: 71.553 - - type: map_at_10 - value: 85.59 - - type: map_at_100 - value: 86.205 - - type: map_at_1000 - value: 86.219 - - type: map_at_20 - value: 85.985 - - type: map_at_3 - value: 82.65400000000001 - - type: map_at_5 - value: 84.514 - - type: mrr_at_1 - value: 82.07 - - type: mrr_at_10 - value: 88.33148015872987 - - type: mrr_at_100 - value: 88.40739111500831 - - type: mrr_at_1000 - value: 88.40805247679367 - - type: mrr_at_20 - value: 88.38813608158237 - - type: mrr_at_3 - value: 87.42999999999971 - - type: mrr_at_5 - value: 88.06649999999964 - - type: nauc_map_at_1000_diff1 - value: 77.08553781000765 - - type: nauc_map_at_1000_max - value: 23.450946013228485 - - type: nauc_map_at_1000_std - value: -35.270774669376145 - - type: nauc_map_at_100_diff1 - value: 77.09137968313824 - - type: nauc_map_at_100_max - value: 23.4177325252657 - - type: nauc_map_at_100_std - value: -35.29868072622312 - - type: nauc_map_at_10_diff1 - value: 77.35520846930982 - - type: nauc_map_at_10_max - value: 22.722710833976105 - - type: nauc_map_at_10_std - value: -37.07802906799404 - - type: nauc_map_at_1_diff1 - value: 79.44035377028345 - - type: nauc_map_at_1_max - value: 18.453731579204632 - - type: nauc_map_at_1_std - value: -31.459193294191756 - - type: nauc_map_at_20_diff1 - value: 77.20665962095318 - - type: nauc_map_at_20_max - value: 23.121398453381502 - - type: nauc_map_at_20_std - value: -36.048384547201145 - - type: nauc_map_at_3_diff1 - value: 77.73189801819204 - - type: nauc_map_at_3_max - value: 20.89861737342013 - - type: nauc_map_at_3_std - value: -38.69750583432779 - - type: nauc_map_at_5_diff1 - value: 77.38414220339891 - - type: nauc_map_at_5_max - value: 21.745210492173754 - - type: nauc_map_at_5_std - value: -38.2621808062194 - - type: nauc_mrr_at_1000_diff1 - value: 77.6983519127067 - - type: nauc_mrr_at_1000_max - value: 24.940664735847122 - - type: nauc_mrr_at_1000_std - value: -31.766166525611034 - - type: nauc_mrr_at_100_diff1 - value: 77.69677361244631 - - type: nauc_mrr_at_100_max - value: 24.93944911995355 - - type: nauc_mrr_at_100_std - value: -31.766671314071193 - - type: nauc_mrr_at_10_diff1 - value: 77.69969707498353 - - type: nauc_mrr_at_10_max - value: 24.97383508222681 - - type: nauc_mrr_at_10_std - value: -31.926609297621596 - - type: nauc_mrr_at_1_diff1 - value: 78.2888304372505 - - type: nauc_mrr_at_1_max - value: 25.016373889546283 - - type: nauc_mrr_at_1_std - value: -29.403166889867382 - - type: nauc_mrr_at_20_diff1 - value: 77.70610400894614 - - type: nauc_mrr_at_20_max - value: 24.971056592872422 - - type: nauc_mrr_at_20_std - value: -31.78902173656083 - - type: nauc_mrr_at_3_diff1 - value: 77.62843372946364 - - type: nauc_mrr_at_3_max - value: 24.926476114034067 - - type: nauc_mrr_at_3_std - value: -32.40860325018437 - - type: nauc_mrr_at_5_diff1 - value: 77.66814059985857 - - type: nauc_mrr_at_5_max - value: 24.835531623050706 - - type: nauc_mrr_at_5_std - value: -32.163749303020865 - - type: nauc_ndcg_at_1000_diff1 - value: 76.96777944447858 - - type: nauc_ndcg_at_1000_max - value: 24.21885484734531 - - type: nauc_ndcg_at_1000_std - value: -33.74247891547267 - - type: nauc_ndcg_at_100_diff1 - value: 76.9482600352606 - - type: nauc_ndcg_at_100_max - value: 24.0598796503349 - - type: nauc_ndcg_at_100_std - value: -33.778035513403985 - - type: nauc_ndcg_at_10_diff1 - value: 77.22805292790589 - - type: nauc_ndcg_at_10_max - value: 23.864409616522124 - - type: nauc_ndcg_at_10_std - value: -36.37251390369501 - - type: nauc_ndcg_at_1_diff1 - value: 78.32661106648008 - - type: nauc_ndcg_at_1_max - value: 25.084766818780373 - - type: nauc_ndcg_at_1_std - value: -29.468568226177403 - - type: nauc_ndcg_at_20_diff1 - value: 77.16551935289819 - - type: nauc_ndcg_at_20_max - value: 24.070148451760634 - - type: nauc_ndcg_at_20_std - value: -35.07326418566946 - - type: nauc_ndcg_at_3_diff1 - value: 76.93582386618357 - - type: nauc_ndcg_at_3_max - value: 22.831832375706316 - - type: nauc_ndcg_at_3_std - value: -37.193882195851025 - - type: nauc_ndcg_at_5_diff1 - value: 77.01577617426591 - - type: nauc_ndcg_at_5_max - value: 22.91173362852854 - - type: nauc_ndcg_at_5_std - value: -37.292419263723644 - - type: nauc_precision_at_1000_diff1 - value: -44.12943949966939 - - type: nauc_precision_at_1000_max - value: 0.36459404941215495 - - type: nauc_precision_at_1000_std - value: 31.380531983454212 - - type: nauc_precision_at_100_diff1 - value: -43.811695386130445 - - type: nauc_precision_at_100_max - value: 0.03794917738746938 - - type: nauc_precision_at_100_std - value: 31.364315473478683 - - type: nauc_precision_at_10_diff1 - value: -38.68336391743595 - - type: nauc_precision_at_10_max - value: -0.21210695423971426 - - type: nauc_precision_at_10_std - value: 21.30974035423888 - - type: nauc_precision_at_1_diff1 - value: 78.32661106648008 - - type: nauc_precision_at_1_max - value: 25.084766818780373 - - type: nauc_precision_at_1_std - value: -29.468568226177403 - - type: nauc_precision_at_20_diff1 - value: -41.61475122844354 - - type: nauc_precision_at_20_max - value: -0.6430570950832868 - - type: nauc_precision_at_20_std - value: 26.507271600025277 - - type: nauc_precision_at_3_diff1 - value: -18.499808457749403 - - type: nauc_precision_at_3_max - value: 4.1065933271808746 - - type: nauc_precision_at_3_std - value: 3.7205560449518087 - - type: nauc_precision_at_5_diff1 - value: -31.49822345207351 - - type: nauc_precision_at_5_max - value: 1.0136598528061673 - - type: nauc_precision_at_5_std - value: 13.31511595759256 - - type: nauc_recall_at_1000_diff1 - value: 48.292440885754786 - - type: nauc_recall_at_1000_max - value: -6.0463626986179095 - - type: nauc_recall_at_1000_std - value: 15.224554044983163 - - type: nauc_recall_at_100_diff1 - value: 65.28632210066016 - - type: nauc_recall_at_100_max - value: 7.887343144626012 - - type: nauc_recall_at_100_std - value: -27.506802657962965 - - type: nauc_recall_at_10_diff1 - value: 74.5101806742001 - - type: nauc_recall_at_10_max - value: 19.707577867857673 - - type: nauc_recall_at_10_std - value: -53.480724171417926 - - type: nauc_recall_at_1_diff1 - value: 79.44035377028345 - - type: nauc_recall_at_1_max - value: 18.453731579204632 - - type: nauc_recall_at_1_std - value: -31.459193294191756 - - type: nauc_recall_at_20_diff1 - value: 74.96479880287762 - - type: nauc_recall_at_20_max - value: 21.504395275855938 - - type: nauc_recall_at_20_std - value: -48.242306432375024 - - type: nauc_recall_at_3_diff1 - value: 74.5629712077306 - - type: nauc_recall_at_3_max - value: 17.246905733933698 - - type: nauc_recall_at_3_std - value: -46.4441514305998 - - type: nauc_recall_at_5_diff1 - value: 72.96056861757357 - - type: nauc_recall_at_5_max - value: 15.986882659418775 - - type: nauc_recall_at_5_std - value: -50.26880302584927 - - type: ndcg_at_1 - value: 82.05 - - type: ndcg_at_10 - value: 89.237 - - type: ndcg_at_100 - value: 90.316 - - type: ndcg_at_1000 - value: 90.39500000000001 - - type: ndcg_at_20 - value: 89.819 - - type: ndcg_at_3 - value: 86.449 - - type: ndcg_at_5 - value: 88.015 - - type: precision_at_1 - value: 82.05 - - type: precision_at_10 - value: 13.505 - - type: precision_at_100 - value: 1.538 - - type: precision_at_1000 - value: 0.157 - - type: precision_at_20 - value: 7.156 - - type: precision_at_3 - value: 37.730000000000004 - - type: precision_at_5 - value: 24.792 - - type: recall_at_1 - value: 71.553 - - type: recall_at_10 - value: 96.209 - - type: recall_at_100 - value: 99.65899999999999 - - type: recall_at_1000 - value: 99.992 - - type: recall_at_20 - value: 97.99600000000001 - - type: recall_at_3 - value: 88.315 - - type: recall_at_5 - value: 92.713 - task: - type: Retrieval - - dataset: - config: default - name: MTEB RUParaPhraserSTS (default) - revision: 43265056790b8f7c59e0139acb4be0a8dad2c8f4 - split: test - type: merionum/ru_paraphraser - metrics: - - type: cosine_pearson - value: 59.90108783647319 - - type: cosine_spearman - value: 70.47034713463798 - - type: euclidean_pearson - value: 66.01030572916656 - - type: euclidean_spearman - value: 70.47034713463798 - - type: main_score - value: 70.47034713463798 - - type: manhattan_pearson - value: 65.98001943092828 - - type: manhattan_spearman - value: 70.42617367089709 - - type: pearson - value: 59.90108783647319 - - type: spearman - value: 70.47034713463798 - task: - type: STS - - dataset: - config: default - name: MTEB RiaNewsRetrieval (default) - revision: 82374b0bbacda6114f39ff9c5b925fa1512ca5d7 - split: test - type: ai-forever/ria-news-retrieval - metrics: - - type: main_score - value: 71.652 - - type: map_at_1 - value: 56.38999999999999 - - type: map_at_10 - value: 66.86 - - type: map_at_100 - value: 67.257 - - type: map_at_1000 - value: 67.27 - - type: map_at_20 - value: 67.12 - - type: map_at_3 - value: 64.823 - - type: map_at_5 - value: 66.074 - - type: mrr_at_1 - value: 56.38999999999999 - - type: mrr_at_10 - value: 66.86036507936463 - - type: mrr_at_100 - value: 67.25678485069567 - - type: mrr_at_1000 - value: 67.2702075790664 - - type: mrr_at_20 - value: 67.12014471381676 - - type: mrr_at_3 - value: 64.82333333333293 - - type: mrr_at_5 - value: 66.07433333333262 - - type: nauc_map_at_1000_diff1 - value: 61.74297820394619 - - type: nauc_map_at_1000_max - value: 9.118854001349124 - - type: nauc_map_at_1000_std - value: -4.94356578723955 - - type: nauc_map_at_100_diff1 - value: 61.738053783409406 - - type: nauc_map_at_100_max - value: 9.11243471334275 - - type: nauc_map_at_100_std - value: -4.929694246493183 - - type: nauc_map_at_10_diff1 - value: 61.6542231505168 - - type: nauc_map_at_10_max - value: 9.039711080932504 - - type: nauc_map_at_10_std - value: -5.017750438974048 - - type: nauc_map_at_1_diff1 - value: 65.75155095156909 - - type: nauc_map_at_1_max - value: 9.967117075442566 - - type: nauc_map_at_1_std - value: -6.898771142001273 - - type: nauc_map_at_20_diff1 - value: 61.672262441303126 - - type: nauc_map_at_20_max - value: 9.072225719428028 - - type: nauc_map_at_20_std - value: -4.934926207117015 - - type: nauc_map_at_3_diff1 - value: 61.55367486454476 - - type: nauc_map_at_3_max - value: 9.032982079978618 - - type: nauc_map_at_3_std - value: -5.87535764632397 - - type: nauc_map_at_5_diff1 - value: 61.42683855708546 - - type: nauc_map_at_5_max - value: 8.877070831850263 - - type: nauc_map_at_5_std - value: -5.432276887529017 - - type: nauc_mrr_at_1000_diff1 - value: 61.74297820394619 - - type: nauc_mrr_at_1000_max - value: 9.118854001349124 - - type: nauc_mrr_at_1000_std - value: -4.94356578723955 - - type: nauc_mrr_at_100_diff1 - value: 61.738053783409406 - - type: nauc_mrr_at_100_max - value: 9.11243471334275 - - type: nauc_mrr_at_100_std - value: -4.929694246493183 - - type: nauc_mrr_at_10_diff1 - value: 61.6542231505168 - - type: nauc_mrr_at_10_max - value: 9.039711080932504 - - type: nauc_mrr_at_10_std - value: -5.017750438974048 - - type: nauc_mrr_at_1_diff1 - value: 65.75155095156909 - - type: nauc_mrr_at_1_max - value: 9.967117075442566 - - type: nauc_mrr_at_1_std - value: -6.898771142001273 - - type: nauc_mrr_at_20_diff1 - value: 61.672262441303126 - - type: nauc_mrr_at_20_max - value: 9.072225719428028 - - type: nauc_mrr_at_20_std - value: -4.934926207117015 - - type: nauc_mrr_at_3_diff1 - value: 61.55367486454476 - - type: nauc_mrr_at_3_max - value: 9.032982079978618 - - type: nauc_mrr_at_3_std - value: -5.87535764632397 - - type: nauc_mrr_at_5_diff1 - value: 61.42683855708546 - - type: nauc_mrr_at_5_max - value: 8.877070831850263 - - type: nauc_mrr_at_5_std - value: -5.432276887529017 - - type: nauc_ndcg_at_1000_diff1 - value: 60.98630459378577 - - type: nauc_ndcg_at_1000_max - value: 9.246035024166295 - - type: nauc_ndcg_at_1000_std - value: -3.361612562569219 - - type: nauc_ndcg_at_100_diff1 - value: 60.84577656910963 - - type: nauc_ndcg_at_100_max - value: 9.075671762808687 - - type: nauc_ndcg_at_100_std - value: -2.862068237853277 - - type: nauc_ndcg_at_10_diff1 - value: 60.31150294618889 - - type: nauc_ndcg_at_10_max - value: 8.70551760752817 - - type: nauc_ndcg_at_10_std - value: -3.370467925663366 - - type: nauc_ndcg_at_1_diff1 - value: 65.75155095156909 - - type: nauc_ndcg_at_1_max - value: 9.967117075442566 - - type: nauc_ndcg_at_1_std - value: -6.898771142001273 - - type: nauc_ndcg_at_20_diff1 - value: 60.37124365840994 - - type: nauc_ndcg_at_20_max - value: 8.823506745258438 - - type: nauc_ndcg_at_20_std - value: -2.940751680095269 - - type: nauc_ndcg_at_3_diff1 - value: 60.141368320144196 - - type: nauc_ndcg_at_3_max - value: 8.68028808058021 - - type: nauc_ndcg_at_3_std - value: -5.451032717174482 - - type: nauc_ndcg_at_5_diff1 - value: 59.79679000319666 - - type: nauc_ndcg_at_5_max - value: 8.363528454727524 - - type: nauc_ndcg_at_5_std - value: -4.548703878489744 - - type: nauc_precision_at_1000_diff1 - value: 50.710404643785125 - - type: nauc_precision_at_1000_max - value: 30.396375417637838 - - type: nauc_precision_at_1000_std - value: 54.40838986196943 - - type: nauc_precision_at_100_diff1 - value: 52.00580603436627 - - type: nauc_precision_at_100_max - value: 10.99642529617639 - - type: nauc_precision_at_100_std - value: 37.658017921348645 - - type: nauc_precision_at_10_diff1 - value: 52.849600810231344 - - type: nauc_precision_at_10_max - value: 6.807738858421414 - - type: nauc_precision_at_10_std - value: 7.489611728241942 - - type: nauc_precision_at_1_diff1 - value: 65.75155095156909 - - type: nauc_precision_at_1_max - value: 9.967117075442566 - - type: nauc_precision_at_1_std - value: -6.898771142001273 - - type: nauc_precision_at_20_diff1 - value: 51.02573126744686 - - type: nauc_precision_at_20_max - value: 7.205985724189931 - - type: nauc_precision_at_20_std - value: 15.162254397124517 - - type: nauc_precision_at_3_diff1 - value: 55.04812303914477 - - type: nauc_precision_at_3_max - value: 7.391536062448881 - - type: nauc_precision_at_3_std - value: -3.8857451492220694 - - type: nauc_precision_at_5_diff1 - value: 52.748692451066916 - - type: nauc_precision_at_5_max - value: 6.0846073222739605 - - type: nauc_precision_at_5_std - value: -0.5062395770410055 - - type: nauc_recall_at_1000_diff1 - value: 50.710404643785445 - - type: nauc_recall_at_1000_max - value: 30.396375417637206 - - type: nauc_recall_at_1000_std - value: 54.4083898619685 - - type: nauc_recall_at_100_diff1 - value: 52.00580603436653 - - type: nauc_recall_at_100_max - value: 10.996425296176398 - - type: nauc_recall_at_100_std - value: 37.658017921348176 - - type: nauc_recall_at_10_diff1 - value: 52.84960081023129 - - type: nauc_recall_at_10_max - value: 6.807738858421314 - - type: nauc_recall_at_10_std - value: 7.4896117282420684 - - type: nauc_recall_at_1_diff1 - value: 65.75155095156909 - - type: nauc_recall_at_1_max - value: 9.967117075442566 - - type: nauc_recall_at_1_std - value: -6.898771142001273 - - type: nauc_recall_at_20_diff1 - value: 51.025731267446794 - - type: nauc_recall_at_20_max - value: 7.2059857241901035 - - type: nauc_recall_at_20_std - value: 15.162254397124528 - - type: nauc_recall_at_3_diff1 - value: 55.04812303914475 - - type: nauc_recall_at_3_max - value: 7.39153606244884 - - type: nauc_recall_at_3_std - value: -3.8857451492221013 - - type: nauc_recall_at_5_diff1 - value: 52.748692451066994 - - type: nauc_recall_at_5_max - value: 6.08460732227402 - - type: nauc_recall_at_5_std - value: -0.5062395770408601 - - type: ndcg_at_1 - value: 56.38999999999999 - - type: ndcg_at_10 - value: 71.652 - - type: ndcg_at_100 - value: 73.572 - - type: ndcg_at_1000 - value: 73.931 - - type: ndcg_at_20 - value: 72.598 - - type: ndcg_at_3 - value: 67.499 - - type: ndcg_at_5 - value: 69.755 - - type: precision_at_1 - value: 56.38999999999999 - - type: precision_at_10 - value: 8.655 - - type: precision_at_100 - value: 0.955 - - type: precision_at_1000 - value: 0.098 - - type: precision_at_20 - value: 4.515000000000001 - - type: precision_at_3 - value: 25.073 - - type: precision_at_5 - value: 16.14 - - type: recall_at_1 - value: 56.38999999999999 - - type: recall_at_10 - value: 86.55000000000001 - - type: recall_at_100 - value: 95.50999999999999 - - type: recall_at_1000 - value: 98.34 - - type: recall_at_20 - value: 90.29 - - type: recall_at_3 - value: 75.22 - - type: recall_at_5 - value: 80.7 - task: - type: Retrieval - - dataset: - config: default - name: MTEB RuBQReranking (default) - revision: 2e96b8f098fa4b0950fc58eacadeb31c0d0c7fa2 - split: test - type: ai-forever/rubq-reranking - metrics: - - type: main_score - value: 76.66017430254296 - - type: map - value: 76.66017430254296 - - type: mrr - value: 81.53464738571121 - - type: nAUC_map_diff1 - value: 36.278917076250224 - - type: nAUC_map_max - value: 3.203433802863856 - - type: nAUC_map_std - value: 6.996438618963433 - - type: nAUC_mrr_diff1 - value: 42.67990398885311 - - type: nAUC_mrr_max - value: 7.527480599102641 - - type: nAUC_mrr_std - value: 10.097273643521257 - task: - type: Reranking - - dataset: - config: default - name: MTEB RuBQRetrieval (default) - revision: e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b - split: test - type: ai-forever/rubq-retrieval - metrics: - - type: main_score - value: 73.657 - - type: map_at_1 - value: 44.065 - - type: map_at_10 - value: 65.949 - - type: map_at_100 - value: 66.913 - - type: map_at_1000 - value: 66.929 - - type: map_at_20 - value: 66.636 - - type: map_at_3 - value: 60.377 - - type: map_at_5 - value: 64.036 - - type: mrr_at_1 - value: 62.470449172576835 - - type: mrr_at_10 - value: 74.10456302300273 - - type: mrr_at_100 - value: 74.3642061621128 - - type: mrr_at_1000 - value: 74.36885014443637 - - type: mrr_at_20 - value: 74.27597096066316 - - type: mrr_at_3 - value: 72.28132387706853 - - type: mrr_at_5 - value: 73.51950354609932 - - type: nauc_map_at_1000_diff1 - value: 37.090200381007115 - - type: nauc_map_at_1000_max - value: 13.910508855090157 - - type: nauc_map_at_1000_std - value: -18.808239886229163 - - type: nauc_map_at_100_diff1 - value: 37.07829615924866 - - type: nauc_map_at_100_max - value: 13.924480342518827 - - type: nauc_map_at_100_std - value: -18.782458187728533 - - type: nauc_map_at_10_diff1 - value: 36.50603714813933 - - type: nauc_map_at_10_max - value: 13.474364155052655 - - type: nauc_map_at_10_std - value: -19.3217697456299 - - type: nauc_map_at_1_diff1 - value: 42.55523796946059 - - type: nauc_map_at_1_max - value: 8.718694804676934 - - type: nauc_map_at_1_std - value: -16.32083870750557 - - type: nauc_map_at_20_diff1 - value: 36.926962641332715 - - type: nauc_map_at_20_max - value: 13.897116768015918 - - type: nauc_map_at_20_std - value: -18.905045444291904 - - type: nauc_map_at_3_diff1 - value: 36.60441612374597 - - type: nauc_map_at_3_max - value: 11.374638753689874 - - type: nauc_map_at_3_std - value: -18.820705732888126 - - type: nauc_map_at_5_diff1 - value: 36.01240755195151 - - type: nauc_map_at_5_max - value: 12.469431623670511 - - type: nauc_map_at_5_std - value: -19.369051759609658 - - type: nauc_mrr_at_1000_diff1 - value: 46.3294742344679 - - type: nauc_mrr_at_1000_max - value: 17.331355042527218 - - type: nauc_mrr_at_1000_std - value: -22.032497848622864 - - type: nauc_mrr_at_100_diff1 - value: 46.325963720930204 - - type: nauc_mrr_at_100_max - value: 17.34386391231157 - - type: nauc_mrr_at_100_std - value: -22.024696828474614 - - type: nauc_mrr_at_10_diff1 - value: 46.16444815881303 - - type: nauc_mrr_at_10_max - value: 17.27456729923913 - - type: nauc_mrr_at_10_std - value: -22.185922618075203 - - type: nauc_mrr_at_1_diff1 - value: 50.41611103393129 - - type: nauc_mrr_at_1_max - value: 16.01922677023717 - - type: nauc_mrr_at_1_std - value: -21.5541331054664 - - type: nauc_mrr_at_20_diff1 - value: 46.26488921954804 - - type: nauc_mrr_at_20_max - value: 17.35070812737816 - - type: nauc_mrr_at_20_std - value: -22.05522849907832 - - type: nauc_mrr_at_3_diff1 - value: 45.91689510009834 - - type: nauc_mrr_at_3_max - value: 17.33603270222595 - - type: nauc_mrr_at_3_std - value: -22.138054343411756 - - type: nauc_mrr_at_5_diff1 - value: 45.95600551670136 - - type: nauc_mrr_at_5_max - value: 16.876408776684325 - - type: nauc_mrr_at_5_std - value: -22.553431986981593 - - type: nauc_ndcg_at_1000_diff1 - value: 38.90982879614141 - - type: nauc_ndcg_at_1000_max - value: 16.062132022098236 - - type: nauc_ndcg_at_1000_std - value: -18.82741879033837 - - type: nauc_ndcg_at_100_diff1 - value: 38.59230622276177 - - type: nauc_ndcg_at_100_max - value: 16.479030069575217 - - type: nauc_ndcg_at_100_std - value: -18.258623331677327 - - type: nauc_ndcg_at_10_diff1 - value: 36.64193020198755 - - type: nauc_ndcg_at_10_max - value: 15.521465758833719 - - type: nauc_ndcg_at_10_std - value: -19.921078208056432 - - type: nauc_ndcg_at_1_diff1 - value: 50.5575101103112 - - type: nauc_ndcg_at_1_max - value: 16.172381702653436 - - type: nauc_ndcg_at_1_std - value: -21.33447567609692 - - type: nauc_ndcg_at_20_diff1 - value: 37.69618732698285 - - type: nauc_ndcg_at_20_max - value: 16.546221585095637 - - type: nauc_ndcg_at_20_std - value: -18.709641022125947 - - type: nauc_ndcg_at_3_diff1 - value: 37.126400354430665 - - type: nauc_ndcg_at_3_max - value: 12.547246162551406 - - type: nauc_ndcg_at_3_std - value: -20.396178391625853 - - type: nauc_ndcg_at_5_diff1 - value: 35.89350194051627 - - type: nauc_ndcg_at_5_max - value: 13.49687833962685 - - type: nauc_ndcg_at_5_std - value: -20.308812248088064 - - type: nauc_precision_at_1000_diff1 - value: -8.48188084168493 - - type: nauc_precision_at_1000_max - value: 6.139161581867971 - - type: nauc_precision_at_1000_std - value: 4.098842102393299 - - type: nauc_precision_at_100_diff1 - value: -7.885830971996956 - - type: nauc_precision_at_100_max - value: 8.172209543497003 - - type: nauc_precision_at_100_std - value: 5.537025566079311 - - type: nauc_precision_at_10_diff1 - value: -3.866974408944356 - - type: nauc_precision_at_10_max - value: 10.976927790797955 - - type: nauc_precision_at_10_std - value: -1.9349418974770733 - - type: nauc_precision_at_1_diff1 - value: 50.5575101103112 - - type: nauc_precision_at_1_max - value: 16.172381702653436 - - type: nauc_precision_at_1_std - value: -21.33447567609692 - - type: nauc_precision_at_20_diff1 - value: -5.637729717327837 - - type: nauc_precision_at_20_max - value: 10.970959140054813 - - type: nauc_precision_at_20_std - value: 2.8388876862575003 - - type: nauc_precision_at_3_diff1 - value: 9.266489853955846 - - type: nauc_precision_at_3_max - value: 11.106426174438223 - - type: nauc_precision_at_3_std - value: -11.54429747011779 - - type: nauc_precision_at_5_diff1 - value: 0.28893207152211653 - - type: nauc_precision_at_5_max - value: 10.13303315463123 - - type: nauc_precision_at_5_std - value: -6.70447500207753 - - type: nauc_recall_at_1000_diff1 - value: 16.221661299215462 - - type: nauc_recall_at_1000_max - value: 53.597491328642555 - - type: nauc_recall_at_1000_std - value: 64.64136682375486 - - type: nauc_recall_at_100_diff1 - value: 12.993211883064685 - - type: nauc_recall_at_100_max - value: 45.55281725648927 - - type: nauc_recall_at_100_std - value: 26.56347354237723 - - type: nauc_recall_at_10_diff1 - value: 17.77929264056643 - - type: nauc_recall_at_10_max - value: 16.70396492397608 - - type: nauc_recall_at_10_std - value: -17.07872240660004 - - type: nauc_recall_at_1_diff1 - value: 42.55523796946059 - - type: nauc_recall_at_1_max - value: 8.718694804676934 - - type: nauc_recall_at_1_std - value: -16.32083870750557 - - type: nauc_recall_at_20_diff1 - value: 17.12006526746598 - - type: nauc_recall_at_20_max - value: 25.18437803181794 - - type: nauc_recall_at_20_std - value: -7.82129157238653 - - type: nauc_recall_at_3_diff1 - value: 26.51229427509311 - - type: nauc_recall_at_3_max - value: 9.47296410240245 - - type: nauc_recall_at_3_std - value: -17.91249081717094 - - type: nauc_recall_at_5_diff1 - value: 21.037615823181476 - - type: nauc_recall_at_5_max - value: 10.398109095863244 - - type: nauc_recall_at_5_std - value: -18.1775717992346 - - type: ndcg_at_1 - value: 62.41100000000001 - - type: ndcg_at_10 - value: 73.657 - - type: ndcg_at_100 - value: 76.44999999999999 - - type: ndcg_at_1000 - value: 76.724 - - type: ndcg_at_20 - value: 75.241 - - type: ndcg_at_3 - value: 66.53099999999999 - - type: ndcg_at_5 - value: 70.545 - - type: precision_at_1 - value: 62.41100000000001 - - type: precision_at_10 - value: 14.309 - - type: precision_at_100 - value: 1.6400000000000001 - - type: precision_at_1000 - value: 0.167 - - type: precision_at_20 - value: 7.686 - - type: precision_at_3 - value: 36.131 - - type: precision_at_5 - value: 25.413999999999998 - - type: recall_at_1 - value: 44.065 - - type: recall_at_10 - value: 87.569 - - type: recall_at_100 - value: 97.866 - - type: recall_at_1000 - value: 99.663 - - type: recall_at_20 - value: 92.61399999999999 - - type: recall_at_3 - value: 70.376 - - type: recall_at_5 - value: 79.596 - task: - type: Retrieval - - dataset: - config: default - name: MTEB RuReviewsClassification (default) - revision: f6d2c31f4dc6b88f468552750bfec05b4b41b05a - split: test - type: ai-forever/ru-reviews-classification - metrics: - - type: accuracy - value: 73.7150390625 - - type: f1 - value: 72.2246927657427 - - type: f1_weighted - value: 72.22538385584937 - - type: main_score - value: 73.7150390625 - task: - type: Classification - - dataset: - config: default - name: MTEB RuSTSBenchmarkSTS (default) - revision: 7cf24f325c6da6195df55bef3d86b5e0616f3018 - split: test - type: ai-forever/ru-stsbenchmark-sts - metrics: - - type: cosine_pearson - value: 80.99594416064292 - - type: cosine_spearman - value: 79.30076871318474 - - type: euclidean_pearson - value: 80.07768395803016 - - type: euclidean_spearman - value: 79.30076871318474 - - type: main_score - value: 79.30076871318474 - - type: manhattan_pearson - value: 80.14948537837334 - - type: manhattan_spearman - value: 79.37609905744645 - - type: pearson - value: 80.99594416064292 - - type: spearman - value: 79.30076871318474 - task: - type: STS - - dataset: - config: default - name: MTEB RuSciBenchGRNTIClassification (default) - revision: 673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1 - split: test - type: ai-forever/ru-scibench-grnti-classification - metrics: - - type: accuracy - value: 74.520703125 - - type: f1 - value: 73.07891907358736 - - type: f1_weighted - value: 73.09099552030747 - - type: main_score - value: 74.520703125 - task: - type: Classification - - dataset: - config: default - name: MTEB RuSciBenchGRNTIClusteringP2P (default) - revision: 673a610d6d3dd91a547a0d57ae1b56f37ebbf6a1 - split: test - type: ai-forever/ru-scibench-grnti-classification - metrics: - - type: main_score - value: 67.3025150400559 - - type: v_measure - value: 67.3025150400559 - - type: v_measure_std - value: 1.005082206057355 - task: - type: Clustering - - dataset: - config: default - name: MTEB RuSciBenchOECDClassification (default) - revision: 26c88e99dcaba32bb45d0e1bfc21902337f6d471 - split: test - type: ai-forever/ru-scibench-oecd-classification - metrics: - - type: accuracy - value: 61.512890625000004 - - type: f1 - value: 59.40504694034752 - - type: f1_weighted - value: 59.40999969456683 - - type: main_score - value: 61.512890625000004 - task: - type: Classification - - dataset: - config: default - name: MTEB RuSciBenchOECDClusteringP2P (default) - revision: 26c88e99dcaba32bb45d0e1bfc21902337f6d471 - split: test - type: ai-forever/ru-scibench-oecd-classification - metrics: - - type: main_score - value: 56.69474614993359 - - type: v_measure - value: 56.69474614993359 - - type: v_measure_std - value: 0.5136380257006159 - task: - type: Clustering - - dataset: - config: default - name: MTEB SCIDOCS (default) - revision: f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88 - split: test - type: mteb/scidocs - metrics: - - type: main_score - value: 19.162000000000003 - - type: map_at_1 - value: 4.475 - - type: map_at_10 - value: 11.306 - - type: map_at_100 - value: 13.628000000000002 - - type: map_at_1000 - value: 13.977 - - type: map_at_20 - value: 12.41 - - type: map_at_3 - value: 7.941 - - type: map_at_5 - value: 9.681 - - type: mrr_at_1 - value: 22.2 - - type: mrr_at_10 - value: 32.70718253968251 - - type: mrr_at_100 - value: 33.944931964999924 - - type: mrr_at_1000 - value: 33.99928862072715 - - type: mrr_at_20 - value: 33.421887159967625 - - type: mrr_at_3 - value: 29.03333333333334 - - type: mrr_at_5 - value: 31.3883333333333 - - type: nauc_map_at_1000_diff1 - value: 16.360578088282296 - - type: nauc_map_at_1000_max - value: 24.255546647291375 - - type: nauc_map_at_1000_std - value: 23.653561658845028 - - type: nauc_map_at_100_diff1 - value: 16.194025097940372 - - type: nauc_map_at_100_max - value: 24.085864603626415 - - type: nauc_map_at_100_std - value: 23.364987990124177 - - type: nauc_map_at_10_diff1 - value: 17.166987284706135 - - type: nauc_map_at_10_max - value: 23.9828055787183 - - type: nauc_map_at_10_std - value: 20.94729363065384 - - type: nauc_map_at_1_diff1 - value: 26.524260461587694 - - type: nauc_map_at_1_max - value: 18.237790264396608 - - type: nauc_map_at_1_std - value: 13.862573541948848 - - type: nauc_map_at_20_diff1 - value: 16.714123094555536 - - type: nauc_map_at_20_max - value: 23.718224495718733 - - type: nauc_map_at_20_std - value: 21.870044423697838 - - type: nauc_map_at_3_diff1 - value: 19.512270329933145 - - type: nauc_map_at_3_max - value: 20.855620393884433 - - type: nauc_map_at_3_std - value: 16.035011072648476 - - type: nauc_map_at_5_diff1 - value: 18.82304971891875 - - type: nauc_map_at_5_max - value: 21.734182123122007 - - type: nauc_map_at_5_std - value: 18.04423957871652 - - type: nauc_mrr_at_1000_diff1 - value: 20.80051056261849 - - type: nauc_mrr_at_1000_max - value: 18.295945240932287 - - type: nauc_mrr_at_1000_std - value: 17.28025318943188 - - type: nauc_mrr_at_100_diff1 - value: 20.790178515273826 - - type: nauc_mrr_at_100_max - value: 18.31986327791731 - - type: nauc_mrr_at_100_std - value: 17.320091305114396 - - type: nauc_mrr_at_10_diff1 - value: 20.8526295835465 - - type: nauc_mrr_at_10_max - value: 18.196816934222145 - - type: nauc_mrr_at_10_std - value: 17.068040995717965 - - type: nauc_mrr_at_1_diff1 - value: 26.440272836765473 - - type: nauc_mrr_at_1_max - value: 18.049427668224048 - - type: nauc_mrr_at_1_std - value: 13.892939744935399 - - type: nauc_mrr_at_20_diff1 - value: 20.74305901889222 - - type: nauc_mrr_at_20_max - value: 18.201319332283358 - - type: nauc_mrr_at_20_std - value: 17.323956767473867 - - type: nauc_mrr_at_3_diff1 - value: 20.80860026418777 - - type: nauc_mrr_at_3_max - value: 17.93980894932549 - - type: nauc_mrr_at_3_std - value: 16.018953507592425 - - type: nauc_mrr_at_5_diff1 - value: 20.664338981633467 - - type: nauc_mrr_at_5_max - value: 17.99219934317394 - - type: nauc_mrr_at_5_std - value: 16.87297735676564 - - type: nauc_ndcg_at_1000_diff1 - value: 15.863416705505543 - - type: nauc_ndcg_at_1000_max - value: 24.110686464543267 - - type: nauc_ndcg_at_1000_std - value: 28.286385566558142 - - type: nauc_ndcg_at_100_diff1 - value: 14.680972591439675 - - type: nauc_ndcg_at_100_max - value: 23.239600232159475 - - type: nauc_ndcg_at_100_std - value: 27.111251722669195 - - type: nauc_ndcg_at_10_diff1 - value: 16.420888690623528 - - type: nauc_ndcg_at_10_max - value: 22.751709784752375 - - type: nauc_ndcg_at_10_std - value: 21.453901377928283 - - type: nauc_ndcg_at_1_diff1 - value: 26.440272836765473 - - type: nauc_ndcg_at_1_max - value: 18.049427668224048 - - type: nauc_ndcg_at_1_std - value: 13.892939744935399 - - type: nauc_ndcg_at_20_diff1 - value: 15.811379241678267 - - type: nauc_ndcg_at_20_max - value: 22.240678546737126 - - type: nauc_ndcg_at_20_std - value: 23.198241749302497 - - type: nauc_ndcg_at_3_diff1 - value: 18.504832287432112 - - type: nauc_ndcg_at_3_max - value: 19.752235860773588 - - type: nauc_ndcg_at_3_std - value: 16.473921583621845 - - type: nauc_ndcg_at_5_diff1 - value: 17.904056465395726 - - type: nauc_ndcg_at_5_max - value: 20.684461097896246 - - type: nauc_ndcg_at_5_std - value: 18.588666225039148 - - type: nauc_precision_at_1000_diff1 - value: 6.193891363199702 - - type: nauc_precision_at_1000_max - value: 20.852873111518612 - - type: nauc_precision_at_1000_std - value: 37.53891581521263 - - type: nauc_precision_at_100_diff1 - value: 5.97130047912924 - - type: nauc_precision_at_100_max - value: 20.80641359454268 - - type: nauc_precision_at_100_std - value: 33.400028161190946 - - type: nauc_precision_at_10_diff1 - value: 11.257140852641278 - - type: nauc_precision_at_10_max - value: 23.40935798614799 - - type: nauc_precision_at_10_std - value: 24.02179217475008 - - type: nauc_precision_at_1_diff1 - value: 26.440272836765473 - - type: nauc_precision_at_1_max - value: 18.049427668224048 - - type: nauc_precision_at_1_std - value: 13.892939744935399 - - type: nauc_precision_at_20_diff1 - value: 9.777581420498402 - - type: nauc_precision_at_20_max - value: 20.94783417188155 - - type: nauc_precision_at_20_std - value: 26.155229221480035 - - type: nauc_precision_at_3_diff1 - value: 14.815195778747578 - - type: nauc_precision_at_3_max - value: 20.297896590597084 - - type: nauc_precision_at_3_std - value: 17.27297537409199 - - type: nauc_precision_at_5_diff1 - value: 13.929822537338527 - - type: nauc_precision_at_5_max - value: 20.98801445292839 - - type: nauc_precision_at_5_std - value: 20.36762932119487 - - type: nauc_recall_at_1000_diff1 - value: 6.269540116426191 - - type: nauc_recall_at_1000_max - value: 22.50245807435009 - - type: nauc_recall_at_1000_std - value: 38.92843130214358 - - type: nauc_recall_at_100_diff1 - value: 5.858562324196213 - - type: nauc_recall_at_100_max - value: 21.146719698636726 - - type: nauc_recall_at_100_std - value: 33.409178845783586 - - type: nauc_recall_at_10_diff1 - value: 10.791079221016838 - - type: nauc_recall_at_10_max - value: 23.402958626226223 - - type: nauc_recall_at_10_std - value: 23.778905836972058 - - type: nauc_recall_at_1_diff1 - value: 26.524260461587694 - - type: nauc_recall_at_1_max - value: 18.237790264396608 - - type: nauc_recall_at_1_std - value: 13.862573541948848 - - type: nauc_recall_at_20_diff1 - value: 9.588806834662645 - - type: nauc_recall_at_20_max - value: 20.992020581818323 - - type: nauc_recall_at_20_std - value: 25.92296591496667 - - type: nauc_recall_at_3_diff1 - value: 14.86565067742853 - - type: nauc_recall_at_3_max - value: 20.312249816233308 - - type: nauc_recall_at_3_std - value: 16.924697629563205 - - type: nauc_recall_at_5_diff1 - value: 13.735197736933962 - - type: nauc_recall_at_5_max - value: 20.94698008166532 - - type: nauc_recall_at_5_std - value: 19.962609439107787 - - type: ndcg_at_1 - value: 22.2 - - type: ndcg_at_10 - value: 19.162000000000003 - - type: ndcg_at_100 - value: 28.321 - - type: ndcg_at_1000 - value: 34.266999999999996 - - type: ndcg_at_20 - value: 22.253999999999998 - - type: ndcg_at_3 - value: 17.798 - - type: ndcg_at_5 - value: 15.975 - - type: precision_at_1 - value: 22.2 - - type: precision_at_10 - value: 9.98 - - type: precision_at_100 - value: 2.329 - - type: precision_at_1000 - value: 0.376 - - type: precision_at_20 - value: 6.795 - - type: precision_at_3 - value: 16.567 - - type: precision_at_5 - value: 14.24 - - type: recall_at_1 - value: 4.475 - - type: recall_at_10 - value: 20.307 - - type: recall_at_100 - value: 47.242 - - type: recall_at_1000 - value: 76.242 - - type: recall_at_20 - value: 27.596999999999998 - - type: recall_at_3 - value: 10.05 - - type: recall_at_5 - value: 14.432 - task: - type: Retrieval - - dataset: - config: default - name: MTEB SICK-R (default) - revision: 20a6d6f312dd54037fe07a32d58e5e168867909d - split: test - type: mteb/sickr-sts - metrics: - - type: cosine_pearson - value: 85.571141897898 - - type: cosine_spearman - value: 81.06253068558962 - - type: euclidean_pearson - value: 83.27075641353736 - - type: euclidean_spearman - value: 81.06253622684207 - - type: main_score - value: 81.06253068558962 - - type: manhattan_pearson - value: 83.35659901147551 - - type: manhattan_spearman - value: 81.13591158643307 - - type: pearson - value: 85.571141897898 - - type: spearman - value: 81.06253068558962 - task: - type: STS - - dataset: - config: default - name: MTEB STS12 (default) - revision: a0d554a64d88156834ff5ae9920b964011b16384 - split: test - type: mteb/sts12-sts - metrics: - - type: cosine_pearson - value: 81.78267795485768 - - type: cosine_spearman - value: 70.09877188862463 - - type: euclidean_pearson - value: 78.49440835422978 - - type: euclidean_spearman - value: 70.09877188862463 - - type: main_score - value: 70.09877188862463 - - type: manhattan_pearson - value: 78.7282306108082 - - type: manhattan_spearman - value: 70.29557277377175 - - type: pearson - value: 81.78267795485768 - - type: spearman - value: 70.09877188862463 - task: - type: STS - - dataset: - config: default - name: MTEB STS13 (default) - revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca - split: test - type: mteb/sts13-sts - metrics: - - type: cosine_pearson - value: 80.25745663712165 - - type: cosine_spearman - value: 81.01483036919429 - - type: euclidean_pearson - value: 81.14541657219165 - - type: euclidean_spearman - value: 81.01478607738545 - - type: main_score - value: 81.01483036919429 - - type: manhattan_pearson - value: 81.40111412290987 - - type: manhattan_spearman - value: 81.28649812590226 - - type: pearson - value: 80.25745663712165 - - type: spearman - value: 81.01483036919429 - task: - type: STS - - dataset: - config: default - name: MTEB STS14 (default) - revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 - split: test - type: mteb/sts14-sts - metrics: - - type: cosine_pearson - value: 81.7451052491344 - - type: cosine_spearman - value: 77.81666029040504 - - type: euclidean_pearson - value: 81.31747977745773 - - type: euclidean_spearman - value: 77.81666029040504 - - type: main_score - value: 77.81666029040504 - - type: manhattan_pearson - value: 81.46161935201552 - - type: manhattan_spearman - value: 77.98131880904403 - - type: pearson - value: 81.7451052491344 - - type: spearman - value: 77.81666029040504 - task: - type: STS - - dataset: - config: default - name: MTEB STS15 (default) - revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 - split: test - type: mteb/sts15-sts - metrics: - - type: cosine_pearson - value: 83.69084648758516 - - type: cosine_spearman - value: 84.52134445704115 - - type: euclidean_pearson - value: 84.13552443709717 - - type: euclidean_spearman - value: 84.52134445704115 - - type: main_score - value: 84.52134445704115 - - type: manhattan_pearson - value: 84.21966806051941 - - type: manhattan_spearman - value: 84.61306373914836 - - type: pearson - value: 83.69084648758516 - - type: spearman - value: 84.52134445704115 - task: - type: STS - - dataset: - config: default - name: MTEB STS16 (default) - revision: 4d8694f8f0e0100860b497b999b3dbed754a0513 - split: test - type: mteb/sts16-sts - metrics: - - type: cosine_pearson - value: 79.22411263296355 - - type: cosine_spearman - value: 81.0920748151385 - - type: euclidean_pearson - value: 80.69903573722831 - - type: euclidean_spearman - value: 81.0920748151385 - - type: main_score - value: 81.0920748151385 - - type: manhattan_pearson - value: 80.82176565456413 - - type: manhattan_spearman - value: 81.22812310589154 - - type: pearson - value: 79.22411263296355 - - type: spearman - value: 81.0920748151385 - task: - type: STS - - dataset: - config: ru - name: MTEB STS22 (ru) - revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 - split: test - type: mteb/sts22-crosslingual-sts - metrics: - - type: cosine_pearson - value: 46.40454184212831 - - type: cosine_spearman - value: 61.10568034074938 - - type: euclidean_pearson - value: 51.24169968038525 - - type: euclidean_spearman - value: 61.10568034074938 - - type: main_score - value: 61.10568034074938 - - type: manhattan_pearson - value: 52.48738021500564 - - type: manhattan_spearman - value: 61.15965141864881 - - type: pearson - value: 46.40454184212831 - - type: spearman - value: 61.10568034074938 - task: - type: STS - - dataset: - config: default - name: MTEB STSBenchmark (default) - revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 - split: test - type: mteb/stsbenchmark-sts - metrics: - - type: cosine_pearson - value: 82.25427804989813 - - type: cosine_spearman - value: 82.20135109121625 - - type: euclidean_pearson - value: 83.21943775385887 - - type: euclidean_spearman - value: 82.2013271737365 - - type: main_score - value: 82.20135109121625 - - type: manhattan_pearson - value: 83.30139032750424 - - type: manhattan_spearman - value: 82.313788704099 - - type: pearson - value: 82.25427804989813 - - type: spearman - value: 82.20135109121625 - task: - type: STS - - dataset: - config: default - name: MTEB SciDocsRR (default) - revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab - split: test - type: mteb/scidocs-reranking - metrics: - - type: main_score - value: 88.00790793717358 - - type: map - value: 88.00790793717358 - - type: mrr - value: 96.2317149081855 - - type: nAUC_map_diff1 - value: -4.285706894005613 - - type: nAUC_map_max - value: 47.69800035323553 - - type: nAUC_map_std - value: 66.05509605566249 - - type: nAUC_mrr_diff1 - value: 37.59327273978339 - - type: nAUC_mrr_max - value: 74.83336690604901 - - type: nAUC_mrr_std - value: 79.76400854713901 - task: - type: Reranking - - dataset: - config: default - name: MTEB SciFact (default) - revision: 0228b52cf27578f30900b9e5271d331663a030d7 - split: test - type: mteb/scifact - metrics: - - type: main_score - value: 72.902 - - type: map_at_1 - value: 58.160999999999994 - - type: map_at_10 - value: 68.15 - - type: map_at_100 - value: 68.76299999999999 - - type: map_at_1000 - value: 68.777 - - type: map_at_20 - value: 68.636 - - type: map_at_3 - value: 65.496 - - type: map_at_5 - value: 67.166 - - type: mrr_at_1 - value: 61.66666666666667 - - type: mrr_at_10 - value: 69.90119047619046 - - type: mrr_at_100 - value: 70.35924850016258 - - type: mrr_at_1000 - value: 70.36956277343268 - - type: mrr_at_20 - value: 70.2427525490219 - - type: mrr_at_3 - value: 68.0 - - type: mrr_at_5 - value: 69.23333333333332 - - type: nauc_map_at_1000_diff1 - value: 67.60995698057089 - - type: nauc_map_at_1000_max - value: 46.068809020612704 - - type: nauc_map_at_1000_std - value: 1.1948672845462098 - - type: nauc_map_at_100_diff1 - value: 67.60952762159904 - - type: nauc_map_at_100_max - value: 46.07379744229154 - - type: nauc_map_at_100_std - value: 1.182970323131957 - - type: nauc_map_at_10_diff1 - value: 67.57824321545665 - - type: nauc_map_at_10_max - value: 46.32612750524493 - - type: nauc_map_at_10_std - value: 1.2781320788709092 - - type: nauc_map_at_1_diff1 - value: 70.5130136513353 - - type: nauc_map_at_1_max - value: 40.673408888331544 - - type: nauc_map_at_1_std - value: -1.3408323300249747 - - type: nauc_map_at_20_diff1 - value: 67.52032735442157 - - type: nauc_map_at_20_max - value: 46.12651713618736 - - type: nauc_map_at_20_std - value: 1.1355480686211972 - - type: nauc_map_at_3_diff1 - value: 68.16380401477547 - - type: nauc_map_at_3_max - value: 43.88680563184851 - - type: nauc_map_at_3_std - value: -3.720201195392404 - - type: nauc_map_at_5_diff1 - value: 67.99170514346827 - - type: nauc_map_at_5_max - value: 45.1182341963952 - - type: nauc_map_at_5_std - value: -0.8736684979875234 - - type: nauc_mrr_at_1000_diff1 - value: 65.5906860945105 - - type: nauc_mrr_at_1000_max - value: 47.56601805618958 - - type: nauc_mrr_at_1000_std - value: 5.9136892538197126 - - type: nauc_mrr_at_100_diff1 - value: 65.59130506472648 - - type: nauc_mrr_at_100_max - value: 47.560766539048224 - - type: nauc_mrr_at_100_std - value: 5.886452280327445 - - type: nauc_mrr_at_10_diff1 - value: 65.38789898290564 - - type: nauc_mrr_at_10_max - value: 47.82853216971601 - - type: nauc_mrr_at_10_std - value: 6.362470864688538 - - type: nauc_mrr_at_1_diff1 - value: 67.64455543946382 - - type: nauc_mrr_at_1_max - value: 44.87347417242169 - - type: nauc_mrr_at_1_std - value: 7.4488417726357365 - - type: nauc_mrr_at_20_diff1 - value: 65.47605317912657 - - type: nauc_mrr_at_20_max - value: 47.58709668066147 - - type: nauc_mrr_at_20_std - value: 5.809216165052655 - - type: nauc_mrr_at_3_diff1 - value: 65.77227321919354 - - type: nauc_mrr_at_3_max - value: 47.54231580750389 - - type: nauc_mrr_at_3_std - value: 4.222335116280824 - - type: nauc_mrr_at_5_diff1 - value: 65.54309281879533 - - type: nauc_mrr_at_5_max - value: 47.89198835887271 - - type: nauc_mrr_at_5_std - value: 5.990782526521852 - - type: nauc_ndcg_at_1000_diff1 - value: 66.50501237344044 - - type: nauc_ndcg_at_1000_max - value: 47.89691614927347 - - type: nauc_ndcg_at_1000_std - value: 3.858567312387061 - - type: nauc_ndcg_at_100_diff1 - value: 66.52978786688968 - - type: nauc_ndcg_at_100_max - value: 47.92798566987145 - - type: nauc_ndcg_at_100_std - value: 3.2827316431460147 - - type: nauc_ndcg_at_10_diff1 - value: 65.58633101511163 - - type: nauc_ndcg_at_10_max - value: 48.80288520105782 - - type: nauc_ndcg_at_10_std - value: 3.9764819883604385 - - type: nauc_ndcg_at_1_diff1 - value: 67.64455543946382 - - type: nauc_ndcg_at_1_max - value: 44.87347417242169 - - type: nauc_ndcg_at_1_std - value: 7.4488417726357365 - - type: nauc_ndcg_at_20_diff1 - value: 65.674407420572 - - type: nauc_ndcg_at_20_max - value: 48.11394986776404 - - type: nauc_ndcg_at_20_std - value: 2.711393534038602 - - type: nauc_ndcg_at_3_diff1 - value: 66.5498828618974 - - type: nauc_ndcg_at_3_max - value: 45.78842720237144 - - type: nauc_ndcg_at_3_std - value: -2.524503922872291 - - type: nauc_ndcg_at_5_diff1 - value: 66.51681687231908 - - type: nauc_ndcg_at_5_max - value: 47.28239002104229 - - type: nauc_ndcg_at_5_std - value: 0.8797704241318283 - - type: nauc_precision_at_1000_diff1 - value: -25.278551984459547 - - type: nauc_precision_at_1000_max - value: 19.072121412932663 - - type: nauc_precision_at_1000_std - value: 51.03267260477789 - - type: nauc_precision_at_100_diff1 - value: -17.95505562847295 - - type: nauc_precision_at_100_max - value: 22.786207573322784 - - type: nauc_precision_at_100_std - value: 45.52886915349048 - - type: nauc_precision_at_10_diff1 - value: 7.786315249522727 - - type: nauc_precision_at_10_max - value: 40.45721598510848 - - type: nauc_precision_at_10_std - value: 38.86811809562086 - - type: nauc_precision_at_1_diff1 - value: 67.64455543946382 - - type: nauc_precision_at_1_max - value: 44.87347417242169 - - type: nauc_precision_at_1_std - value: 7.4488417726357365 - - type: nauc_precision_at_20_diff1 - value: -6.7787477098826265 - - type: nauc_precision_at_20_max - value: 29.43636167690474 - - type: nauc_precision_at_20_std - value: 38.48151740489274 - - type: nauc_precision_at_3_diff1 - value: 39.481963258187015 - - type: nauc_precision_at_3_max - value: 42.42369906705573 - - type: nauc_precision_at_3_std - value: 9.531028412147295 - - type: nauc_precision_at_5_diff1 - value: 24.898111796597416 - - type: nauc_precision_at_5_max - value: 41.17351707929381 - - type: nauc_precision_at_5_std - value: 24.90291910370421 - - type: nauc_recall_at_1000_diff1 - value: 70.76330532212897 - - type: nauc_recall_at_1000_max - value: 88.85387488328593 - - type: nauc_recall_at_1000_std - value: 83.25163398692725 - - type: nauc_recall_at_100_diff1 - value: 68.08723489395771 - - type: nauc_recall_at_100_max - value: 63.74994442221321 - - type: nauc_recall_at_100_std - value: 7.758659019163229 - - type: nauc_recall_at_10_diff1 - value: 56.545518621597324 - - type: nauc_recall_at_10_max - value: 56.74870654004741 - - type: nauc_recall_at_10_std - value: 8.701951811447595 - - type: nauc_recall_at_1_diff1 - value: 70.5130136513353 - - type: nauc_recall_at_1_max - value: 40.673408888331544 - - type: nauc_recall_at_1_std - value: -1.3408323300249747 - - type: nauc_recall_at_20_diff1 - value: 54.13998932906485 - - type: nauc_recall_at_20_max - value: 55.454403983815794 - - type: nauc_recall_at_20_std - value: -3.1705274702472694 - - type: nauc_recall_at_3_diff1 - value: 63.260925909299544 - - type: nauc_recall_at_3_max - value: 46.05508565736711 - - type: nauc_recall_at_3_std - value: -10.957671894931114 - - type: nauc_recall_at_5_diff1 - value: 61.62947530846916 - - type: nauc_recall_at_5_max - value: 50.48704479871539 - - type: nauc_recall_at_5_std - value: -1.4810663364085042 - - type: ndcg_at_1 - value: 61.667 - - type: ndcg_at_10 - value: 72.902 - - type: ndcg_at_100 - value: 75.368 - - type: ndcg_at_1000 - value: 75.657 - - type: ndcg_at_20 - value: 74.453 - - type: ndcg_at_3 - value: 68.57 - - type: ndcg_at_5 - value: 70.926 - - type: precision_at_1 - value: 61.667 - - type: precision_at_10 - value: 9.667 - - type: precision_at_100 - value: 1.093 - - type: precision_at_1000 - value: 0.11199999999999999 - - type: precision_at_20 - value: 5.2 - - type: precision_at_3 - value: 27.0 - - type: precision_at_5 - value: 17.8 - - type: recall_at_1 - value: 58.160999999999994 - - type: recall_at_10 - value: 85.68900000000001 - - type: recall_at_100 - value: 96.5 - - type: recall_at_1000 - value: 98.667 - - type: recall_at_20 - value: 91.60000000000001 - - type: recall_at_3 - value: 74.083 - - type: recall_at_5 - value: 80.022 - task: - type: Retrieval - - dataset: - config: default - name: MTEB SensitiveTopicsClassification (default) - revision: 416b34a802308eac30e4192afc0ff99bb8dcc7f2 - split: test - type: ai-forever/sensitive-topics-classification - metrics: - - type: accuracy - value: 30.009765625 - - type: f1 - value: 31.97592344768223 - - type: lrap - value: 46.34419759114506 - - type: main_score - value: 30.009765625 - task: - type: MultilabelClassification - - dataset: - config: default - name: MTEB SprintDuplicateQuestions (default) - revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46 - split: test - type: mteb/sprintduplicatequestions-pairclassification - metrics: - - type: cosine_accuracy - value: 99.6059405940594 - - type: cosine_accuracy_threshold - value: 80.66068887710571 - - type: cosine_ap - value: 86.29950883344696 - - type: cosine_f1 - value: 79.280205655527 - - type: cosine_f1_threshold - value: 80.31163215637207 - - type: cosine_precision - value: 81.58730158730158 - - type: cosine_recall - value: 77.10000000000001 - - type: dot_accuracy - value: 99.6059405940594 - - type: dot_accuracy_threshold - value: 80.66068291664124 - - type: dot_ap - value: 86.29952127770177 - - type: dot_f1 - value: 79.280205655527 - - type: dot_f1_threshold - value: 80.31163215637207 - - type: dot_precision - value: 81.58730158730158 - - type: dot_recall - value: 77.10000000000001 - - type: euclidean_accuracy - value: 99.6059405940594 - - type: euclidean_accuracy_threshold - value: 62.19213604927063 - - type: euclidean_ap - value: 86.29950883344696 - - type: euclidean_f1 - value: 79.280205655527 - - type: euclidean_f1_threshold - value: 62.75087594985962 - - type: euclidean_precision - value: 81.58730158730158 - - type: euclidean_recall - value: 77.10000000000001 - - type: main_score - value: 86.97106165735022 - - type: manhattan_accuracy - value: 99.61683168316831 - - type: manhattan_accuracy_threshold - value: 2190.611457824707 - - type: manhattan_ap - value: 86.97106165735022 - - type: manhattan_f1 - value: 79.76999477260847 - - type: manhattan_f1_threshold - value: 2214.816665649414 - - type: manhattan_precision - value: 83.57064622124864 - - type: manhattan_recall - value: 76.3 - - type: max_accuracy - value: 99.61683168316831 - - type: max_ap - value: 86.97106165735022 - - type: max_f1 - value: 79.76999477260847 - - type: max_precision - value: 83.57064622124864 - - type: max_recall - value: 77.10000000000001 - - type: similarity_accuracy - value: 99.6059405940594 - - type: similarity_accuracy_threshold - value: 80.66068887710571 - - type: similarity_ap - value: 86.29950883344696 - - type: similarity_f1 - value: 79.280205655527 - - type: similarity_f1_threshold - value: 80.31163215637207 - - type: similarity_precision - value: 81.58730158730158 - - type: similarity_recall - value: 77.10000000000001 - task: - type: PairClassification - - dataset: - config: default - name: MTEB StackOverflowDupQuestions (default) - revision: e185fbe320c72810689fc5848eb6114e1ef5ec69 - split: test - type: mteb/stackoverflowdupquestions-reranking - metrics: - - type: main_score - value: 47.7154636461295 - - type: map - value: 47.7154636461295 - - type: mrr - value: 48.28928730583142 - - type: nAUC_map_diff1 - value: 35.220975539918406 - - type: nAUC_map_max - value: 11.301549750395234 - - type: nAUC_map_std - value: 10.780322232649885 - - type: nAUC_mrr_diff1 - value: 36.03068695413537 - - type: nAUC_mrr_max - value: 11.937927361734655 - - type: nAUC_mrr_std - value: 10.594734349234532 - task: - type: Reranking - - dataset: - config: default - name: MTEB SummEval (default) - revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c - split: test - type: mteb/summeval - metrics: - - type: cosine_pearson - value: 29.3884630972395 - - type: cosine_spearman - value: 27.85662716499775 - - type: dot_pearson - value: 29.388462913794346 - - type: dot_spearman - value: 27.888760562108097 - - type: main_score - value: 27.85662716499775 - - type: pearson - value: 29.3884630972395 - - type: spearman - value: 27.85662716499775 - task: - type: Summarization - - dataset: - config: default - name: MTEB TERRa (default) - revision: 7b58f24536063837d644aab9a023c62199b2a612 - split: dev - type: ai-forever/terra-pairclassification - metrics: - - type: cosine_accuracy - value: 60.91205211726385 - - type: cosine_accuracy_threshold - value: 71.89492583274841 - - type: cosine_ap - value: 59.975232226567556 - - type: cosine_f1 - value: 67.8733031674208 - - type: cosine_f1_threshold - value: 47.70580530166626 - - type: cosine_precision - value: 51.903114186851205 - - type: cosine_recall - value: 98.0392156862745 - - type: dot_accuracy - value: 60.91205211726385 - - type: dot_accuracy_threshold - value: 71.89492583274841 - - type: dot_ap - value: 59.975232226567556 - - type: dot_f1 - value: 67.8733031674208 - - type: dot_f1_threshold - value: 47.7058082818985 - - type: dot_precision - value: 51.903114186851205 - - type: dot_recall - value: 98.0392156862745 - - type: euclidean_accuracy - value: 60.91205211726385 - - type: euclidean_accuracy_threshold - value: 74.97339844703674 - - type: euclidean_ap - value: 59.975232226567556 - - type: euclidean_f1 - value: 67.8733031674208 - - type: euclidean_f1_threshold - value: 102.26843357086182 - - type: euclidean_precision - value: 51.903114186851205 - - type: euclidean_recall - value: 98.0392156862745 - - type: main_score - value: 60.456656396523144 - - type: manhattan_accuracy - value: 61.563517915309454 - - type: manhattan_accuracy_threshold - value: 2685.3965759277344 - - type: manhattan_ap - value: 60.456656396523144 - - type: manhattan_f1 - value: 67.72009029345372 - - type: manhattan_f1_threshold - value: 3692.0753479003906 - - type: manhattan_precision - value: 51.724137931034484 - - type: manhattan_recall - value: 98.0392156862745 - - type: max_accuracy - value: 61.563517915309454 - - type: max_ap - value: 60.456656396523144 - - type: max_f1 - value: 67.8733031674208 - - type: max_precision - value: 51.903114186851205 - - type: max_recall - value: 98.0392156862745 - - type: similarity_accuracy - value: 60.91205211726385 - - type: similarity_accuracy_threshold - value: 71.89492583274841 - - type: similarity_ap - value: 59.975232226567556 - - type: similarity_f1 - value: 67.8733031674208 - - type: similarity_f1_threshold - value: 47.70580530166626 - - type: similarity_precision - value: 51.903114186851205 - - type: similarity_recall - value: 98.0392156862745 - task: - type: PairClassification - - dataset: - config: default - name: MTEB TRECCOVID (default) - revision: bb9466bac8153a0349341eb1b22e06409e78ef4e - split: test - type: mteb/trec-covid - metrics: - - type: main_score - value: 68.527 - - type: map_at_1 - value: 0.22799999999999998 - - type: map_at_10 - value: 1.6310000000000002 - - type: map_at_100 - value: 7.9990000000000006 - - type: map_at_1000 - value: 19.209 - - type: map_at_20 - value: 2.6550000000000002 - - type: map_at_3 - value: 0.562 - - type: map_at_5 - value: 0.914 - - type: mrr_at_1 - value: 84.0 - - type: mrr_at_10 - value: 91.66666666666666 - - type: mrr_at_100 - value: 91.66666666666666 - - type: mrr_at_1000 - value: 91.66666666666666 - - type: mrr_at_20 - value: 91.66666666666666 - - type: mrr_at_3 - value: 91.66666666666666 - - type: mrr_at_5 - value: 91.66666666666666 - - type: nauc_map_at_1000_diff1 - value: -24.9289165556578 - - type: nauc_map_at_1000_max - value: 36.82343869931691 - - type: nauc_map_at_1000_std - value: 69.76693302223156 - - type: nauc_map_at_100_diff1 - value: -14.93215489004332 - - type: nauc_map_at_100_max - value: 26.903137343383193 - - type: nauc_map_at_100_std - value: 51.93554226264329 - - type: nauc_map_at_10_diff1 - value: -9.589639096178356 - - type: nauc_map_at_10_max - value: 0.9848575880881552 - - type: nauc_map_at_10_std - value: 11.949732717032333 - - type: nauc_map_at_1_diff1 - value: -0.02833948708031013 - - type: nauc_map_at_1_max - value: -9.58520783682369 - - type: nauc_map_at_1_std - value: -5.49929878351851 - - type: nauc_map_at_20_diff1 - value: -10.488150815671766 - - type: nauc_map_at_20_max - value: 7.636306025857953 - - type: nauc_map_at_20_std - value: 21.697516125766192 - - type: nauc_map_at_3_diff1 - value: -12.564747985952957 - - type: nauc_map_at_3_max - value: -5.974361276237696 - - type: nauc_map_at_3_std - value: 2.104636022300902 - - type: nauc_map_at_5_diff1 - value: -11.586078080097174 - - type: nauc_map_at_5_max - value: -3.6533992703743476 - - type: nauc_map_at_5_std - value: 6.216862673600229 - - type: nauc_mrr_at_1000_diff1 - value: -29.749167156574195 - - type: nauc_mrr_at_1000_max - value: 25.24789339604172 - - type: nauc_mrr_at_1000_std - value: 47.55242014501294 - - type: nauc_mrr_at_100_diff1 - value: -29.749167156574195 - - type: nauc_mrr_at_100_max - value: 25.24789339604172 - - type: nauc_mrr_at_100_std - value: 47.55242014501294 - - type: nauc_mrr_at_10_diff1 - value: -29.749167156574195 - - type: nauc_mrr_at_10_max - value: 25.24789339604172 - - type: nauc_mrr_at_10_std - value: 47.55242014501294 - - type: nauc_mrr_at_1_diff1 - value: -23.429062346588083 - - type: nauc_mrr_at_1_max - value: 23.92611683848804 - - type: nauc_mrr_at_1_std - value: 45.82719685812465 - - type: nauc_mrr_at_20_diff1 - value: -29.749167156574195 - - type: nauc_mrr_at_20_max - value: 25.24789339604172 - - type: nauc_mrr_at_20_std - value: 47.55242014501294 - - type: nauc_mrr_at_3_diff1 - value: -29.749167156574195 - - type: nauc_mrr_at_3_max - value: 25.24789339604172 - - type: nauc_mrr_at_3_std - value: 47.55242014501294 - - type: nauc_mrr_at_5_diff1 - value: -29.749167156574195 - - type: nauc_mrr_at_5_max - value: 25.24789339604172 - - type: nauc_mrr_at_5_std - value: 47.55242014501294 - - type: nauc_ndcg_at_1000_diff1 - value: -22.98398058353528 - - type: nauc_ndcg_at_1000_max - value: 29.7425796111613 - - type: nauc_ndcg_at_1000_std - value: 66.79973979577099 - - type: nauc_ndcg_at_100_diff1 - value: -24.302637816697096 - - type: nauc_ndcg_at_100_max - value: 35.02366417892295 - - type: nauc_ndcg_at_100_std - value: 73.11481424467178 - - type: nauc_ndcg_at_10_diff1 - value: -36.28539128782605 - - type: nauc_ndcg_at_10_max - value: 22.467365655768294 - - type: nauc_ndcg_at_10_std - value: 57.019631548174964 - - type: nauc_ndcg_at_1_diff1 - value: -31.390887834555393 - - type: nauc_ndcg_at_1_max - value: 34.12862364077993 - - type: nauc_ndcg_at_1_std - value: 42.54840979773444 - - type: nauc_ndcg_at_20_diff1 - value: -30.72488816634815 - - type: nauc_ndcg_at_20_max - value: 26.976879472234245 - - type: nauc_ndcg_at_20_std - value: 62.61724664065122 - - type: nauc_ndcg_at_3_diff1 - value: -42.816038008840906 - - type: nauc_ndcg_at_3_max - value: 22.661518863218703 - - type: nauc_ndcg_at_3_std - value: 43.96298336797927 - - type: nauc_ndcg_at_5_diff1 - value: -41.33190790544439 - - type: nauc_ndcg_at_5_max - value: 21.470060099934962 - - type: nauc_ndcg_at_5_std - value: 52.68059068309917 - - type: nauc_precision_at_1000_diff1 - value: -25.175496297929094 - - type: nauc_precision_at_1000_max - value: 31.092099302925973 - - type: nauc_precision_at_1000_std - value: 55.28335657441642 - - type: nauc_precision_at_100_diff1 - value: -23.82235063790865 - - type: nauc_precision_at_100_max - value: 38.82935361601115 - - type: nauc_precision_at_100_std - value: 74.16286052994596 - - type: nauc_precision_at_10_diff1 - value: -31.256678895241897 - - type: nauc_precision_at_10_max - value: 18.62764539335249 - - type: nauc_precision_at_10_std - value: 55.088078791684566 - - type: nauc_precision_at_1_diff1 - value: -23.429062346588083 - - type: nauc_precision_at_1_max - value: 23.92611683848804 - - type: nauc_precision_at_1_std - value: 45.82719685812465 - - type: nauc_precision_at_20_diff1 - value: -25.457473488205647 - - type: nauc_precision_at_20_max - value: 27.56443770951519 - - type: nauc_precision_at_20_std - value: 62.132518431743186 - - type: nauc_precision_at_3_diff1 - value: -43.08727076791597 - - type: nauc_precision_at_3_max - value: 10.02582635030932 - - type: nauc_precision_at_3_std - value: 36.253506995410675 - - type: nauc_precision_at_5_diff1 - value: -40.1801527075686 - - type: nauc_precision_at_5_max - value: 14.947044956076013 - - type: nauc_precision_at_5_std - value: 50.97875933897098 - - type: nauc_recall_at_1000_diff1 - value: -21.578407961752355 - - type: nauc_recall_at_1000_max - value: 26.522130981068624 - - type: nauc_recall_at_1000_std - value: 55.695556653573455 - - type: nauc_recall_at_100_diff1 - value: -10.077178296188565 - - type: nauc_recall_at_100_max - value: 17.0587187509344 - - type: nauc_recall_at_100_std - value: 36.697069293841245 - - type: nauc_recall_at_10_diff1 - value: -6.87042245147553 - - type: nauc_recall_at_10_max - value: -4.148521545209006 - - type: nauc_recall_at_10_std - value: 6.698236085537414 - - type: nauc_recall_at_1_diff1 - value: -0.02833948708031013 - - type: nauc_recall_at_1_max - value: -9.58520783682369 - - type: nauc_recall_at_1_std - value: -5.49929878351851 - - type: nauc_recall_at_20_diff1 - value: -5.820248045074897 - - type: nauc_recall_at_20_max - value: 2.8626171367487934 - - type: nauc_recall_at_20_std - value: 13.449577793915923 - - type: nauc_recall_at_3_diff1 - value: -9.773766102342227 - - type: nauc_recall_at_3_max - value: -5.58368366828728 - - type: nauc_recall_at_3_std - value: 0.10625733818166493 - - type: nauc_recall_at_5_diff1 - value: -7.531560179183814 - - type: nauc_recall_at_5_max - value: -5.10431775402095 - - type: nauc_recall_at_5_std - value: 2.9170078559993304 - - type: ndcg_at_1 - value: 77.0 - - type: ndcg_at_10 - value: 68.527 - - type: ndcg_at_100 - value: 48.405 - - type: ndcg_at_1000 - value: 44.595 - - type: ndcg_at_20 - value: 62.306 - - type: ndcg_at_3 - value: 72.939 - - type: ndcg_at_5 - value: 72.416 - - type: precision_at_1 - value: 84.0 - - type: precision_at_10 - value: 73.0 - - type: precision_at_100 - value: 49.08 - - type: precision_at_1000 - value: 19.836000000000002 - - type: precision_at_20 - value: 64.7 - - type: precision_at_3 - value: 78.0 - - type: precision_at_5 - value: 78.0 - - type: recall_at_1 - value: 0.22799999999999998 - - type: recall_at_10 - value: 1.881 - - type: recall_at_100 - value: 11.863 - - type: recall_at_1000 - value: 42.081 - - type: recall_at_20 - value: 3.2520000000000002 - - type: recall_at_3 - value: 0.5950000000000001 - - type: recall_at_5 - value: 1.016 - task: - type: Retrieval - - dataset: - config: default - name: MTEB Touche2020 (default) - revision: a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f - split: test - type: mteb/touche2020 - metrics: - - type: main_score - value: 13.937 - - type: map_at_1 - value: 1.2930000000000001 - - type: map_at_10 - value: 5.743 - - type: map_at_100 - value: 9.267 - - type: map_at_1000 - value: 10.742 - - type: map_at_20 - value: 7.303 - - type: map_at_3 - value: 2.896 - - type: map_at_5 - value: 3.939 - - type: mrr_at_1 - value: 14.285714285714285 - - type: mrr_at_10 - value: 26.15889212827988 - - type: mrr_at_100 - value: 27.768355317823477 - - type: mrr_at_1000 - value: 27.81151081519716 - - type: mrr_at_20 - value: 27.13341930178665 - - type: mrr_at_3 - value: 22.448979591836736 - - type: mrr_at_5 - value: 25.102040816326532 - - type: nauc_map_at_1000_diff1 - value: -6.418619908635835 - - type: nauc_map_at_1000_max - value: -29.16827935908967 - - type: nauc_map_at_1000_std - value: 8.072487536940496 - - type: nauc_map_at_100_diff1 - value: -4.085439678921278 - - type: nauc_map_at_100_max - value: -30.0503815156051 - - type: nauc_map_at_100_std - value: 0.5940142547175356 - - type: nauc_map_at_10_diff1 - value: -6.494191455409154 - - type: nauc_map_at_10_max - value: -35.69459573312886 - - type: nauc_map_at_10_std - value: -12.779839188453062 - - type: nauc_map_at_1_diff1 - value: -2.3057823615493915 - - type: nauc_map_at_1_max - value: -33.66603052318725 - - type: nauc_map_at_1_std - value: -13.676412668949379 - - type: nauc_map_at_20_diff1 - value: -6.3310356612575545 - - type: nauc_map_at_20_max - value: -32.01339434864334 - - type: nauc_map_at_20_std - value: -6.815509360579602 - - type: nauc_map_at_3_diff1 - value: -1.133178599145889 - - type: nauc_map_at_3_max - value: -29.76043249136521 - - type: nauc_map_at_3_std - value: -19.964108320064188 - - type: nauc_map_at_5_diff1 - value: -3.985229627379757 - - type: nauc_map_at_5_max - value: -31.662188888720827 - - type: nauc_map_at_5_std - value: -17.887594061086986 - - type: nauc_mrr_at_1000_diff1 - value: 13.01193356074134 - - type: nauc_mrr_at_1000_max - value: -22.95219247566709 - - type: nauc_mrr_at_1000_std - value: -4.671643535232005 - - type: nauc_mrr_at_100_diff1 - value: 13.069921779672386 - - type: nauc_mrr_at_100_max - value: -22.883092040889622 - - type: nauc_mrr_at_100_std - value: -4.5162988896312015 - - type: nauc_mrr_at_10_diff1 - value: 11.578681649426107 - - type: nauc_mrr_at_10_max - value: -24.202645693833944 - - type: nauc_mrr_at_10_std - value: -5.24206333818245 - - type: nauc_mrr_at_1_diff1 - value: 13.128690188364892 - - type: nauc_mrr_at_1_max - value: -28.00109204310416 - - type: nauc_mrr_at_1_std - value: -5.361067751504759 - - type: nauc_mrr_at_20_diff1 - value: 12.991169014806243 - - type: nauc_mrr_at_20_max - value: -23.155024458757023 - - type: nauc_mrr_at_20_std - value: -4.6613893251347776 - - type: nauc_mrr_at_3_diff1 - value: 13.913143082955258 - - type: nauc_mrr_at_3_max - value: -22.375537639765625 - - type: nauc_mrr_at_3_std - value: -14.956409173121715 - - type: nauc_mrr_at_5_diff1 - value: 12.901595857173692 - - type: nauc_mrr_at_5_max - value: -23.996408215558894 - - type: nauc_mrr_at_5_std - value: -6.49836542830797 - - type: nauc_ndcg_at_1000_diff1 - value: 5.779990884574556 - - type: nauc_ndcg_at_1000_max - value: -13.966564935992384 - - type: nauc_ndcg_at_1000_std - value: 42.66569470384277 - - type: nauc_ndcg_at_100_diff1 - value: 6.234930855525906 - - type: nauc_ndcg_at_100_max - value: -24.38418504911329 - - type: nauc_ndcg_at_100_std - value: 18.790399332280156 - - type: nauc_ndcg_at_10_diff1 - value: 7.232052069452057 - - type: nauc_ndcg_at_10_max - value: -30.880638325411265 - - type: nauc_ndcg_at_10_std - value: -0.35569871365077027 - - type: nauc_ndcg_at_1_diff1 - value: 16.555725385111923 - - type: nauc_ndcg_at_1_max - value: -26.013786248753345 - - type: nauc_ndcg_at_1_std - value: -3.928475675217327 - - type: nauc_ndcg_at_20_diff1 - value: 5.616659654115893 - - type: nauc_ndcg_at_20_max - value: -30.436511640651524 - - type: nauc_ndcg_at_20_std - value: -0.19102960481939774 - - type: nauc_ndcg_at_3_diff1 - value: 17.03885024922266 - - type: nauc_ndcg_at_3_max - value: -19.055349937590332 - - type: nauc_ndcg_at_3_std - value: -13.228542865846386 - - type: nauc_ndcg_at_5_diff1 - value: 14.801775219525853 - - type: nauc_ndcg_at_5_max - value: -21.24957330205927 - - type: nauc_ndcg_at_5_std - value: -3.6159018353139163 - - type: nauc_precision_at_1000_diff1 - value: 0.7444573789123141 - - type: nauc_precision_at_1000_max - value: 33.69720059833317 - - type: nauc_precision_at_1000_std - value: 49.16550611043211 - - type: nauc_precision_at_100_diff1 - value: -0.19658777720009285 - - type: nauc_precision_at_100_max - value: -2.151987749997591 - - type: nauc_precision_at_100_std - value: 55.99372777913219 - - type: nauc_precision_at_10_diff1 - value: 0.38911540387349786 - - type: nauc_precision_at_10_max - value: -28.20658920522594 - - type: nauc_precision_at_10_std - value: 12.159116774260385 - - type: nauc_precision_at_1_diff1 - value: 13.128690188364892 - - type: nauc_precision_at_1_max - value: -28.00109204310416 - - type: nauc_precision_at_1_std - value: -5.361067751504759 - - type: nauc_precision_at_20_diff1 - value: 1.5463849480628908 - - type: nauc_precision_at_20_max - value: -19.428037373700505 - - type: nauc_precision_at_20_std - value: 17.118725876441907 - - type: nauc_precision_at_3_diff1 - value: 13.572269577844049 - - type: nauc_precision_at_3_max - value: -19.25460974828505 - - type: nauc_precision_at_3_std - value: -16.101483700594603 - - type: nauc_precision_at_5_diff1 - value: 10.44042220817277 - - type: nauc_precision_at_5_max - value: -20.41453930267137 - - type: nauc_precision_at_5_std - value: 1.0506363193249282 - - type: nauc_recall_at_1000_diff1 - value: 5.452672989062937 - - type: nauc_recall_at_1000_max - value: 9.433825474933833 - - type: nauc_recall_at_1000_std - value: 83.07846603694948 - - type: nauc_recall_at_100_diff1 - value: 2.535260598249035 - - type: nauc_recall_at_100_max - value: -21.695436099245338 - - type: nauc_recall_at_100_std - value: 30.34608369582977 - - type: nauc_recall_at_10_diff1 - value: -3.8610239417771135 - - type: nauc_recall_at_10_max - value: -35.92826433381732 - - type: nauc_recall_at_10_std - value: -4.431168646074323 - - type: nauc_recall_at_1_diff1 - value: -2.3057823615493915 - - type: nauc_recall_at_1_max - value: -33.66603052318725 - - type: nauc_recall_at_1_std - value: -13.676412668949379 - - type: nauc_recall_at_20_diff1 - value: -2.0569660419185167 - - type: nauc_recall_at_20_max - value: -31.28343777931168 - - type: nauc_recall_at_20_std - value: 0.732859893313201 - - type: nauc_recall_at_3_diff1 - value: -1.1280951425624475 - - type: nauc_recall_at_3_max - value: -29.014124779660904 - - type: nauc_recall_at_3_std - value: -24.40550848467014 - - type: nauc_recall_at_5_diff1 - value: -0.5347797031107686 - - type: nauc_recall_at_5_max - value: -31.129644592060185 - - type: nauc_recall_at_5_std - value: -14.423630504744494 - - type: ndcg_at_1 - value: 13.264999999999999 - - type: ndcg_at_10 - value: 13.937 - - type: ndcg_at_100 - value: 23.186 - - type: ndcg_at_1000 - value: 36.565 - - type: ndcg_at_20 - value: 15.665999999999999 - - type: ndcg_at_3 - value: 13.664000000000001 - - type: ndcg_at_5 - value: 13.555 - - type: precision_at_1 - value: 14.285999999999998 - - type: precision_at_10 - value: 13.469000000000001 - - type: precision_at_100 - value: 5.061 - - type: precision_at_1000 - value: 1.367 - - type: precision_at_20 - value: 10.918 - - type: precision_at_3 - value: 15.645999999999999 - - type: precision_at_5 - value: 15.101999999999999 - - type: recall_at_1 - value: 1.2930000000000001 - - type: recall_at_10 - value: 11.106 - - type: recall_at_100 - value: 34.054 - - type: recall_at_1000 - value: 74.773 - - type: recall_at_20 - value: 16.54 - - type: recall_at_3 - value: 3.9669999999999996 - - type: recall_at_5 - value: 6.354 - task: - type: Retrieval - - dataset: - config: default - name: MTEB ToxicConversationsClassification (default) - revision: edfaf9da55d3dd50d43143d90c1ac476895ae6de - split: test - type: mteb/toxic_conversations_50k - metrics: - - type: accuracy - value: 75.3662109375 - - type: ap - value: 17.520455604109124 - - type: ap_weighted - value: 17.520455604109124 - - type: f1 - value: 58.701834404711065 - - type: f1_weighted - value: 80.58084750047006 - - type: main_score - value: 75.3662109375 - task: - type: Classification - - dataset: - config: default - name: MTEB TweetSentimentExtractionClassification (default) - revision: d604517c81ca91fe16a244d1248fc021f9ecee7a - split: test - type: mteb/tweet_sentiment_extraction - metrics: - - type: accuracy - value: 67.64855687606112 - - type: f1 - value: 67.95235466328752 - - type: f1_weighted - value: 67.05421511780088 - - type: main_score - value: 67.64855687606112 - task: - type: Classification - - dataset: - config: default - name: MTEB TwitterSemEval2015 (default) - revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 - split: test - type: mteb/twittersemeval2015-pairclassification - metrics: - - type: cosine_accuracy - value: 83.14358943792097 - - type: cosine_accuracy_threshold - value: 82.20133781433105 - - type: cosine_ap - value: 63.42168849627343 - - type: cosine_f1 - value: 60.19512195121952 - - type: cosine_f1_threshold - value: 78.19918394088745 - - type: cosine_precision - value: 55.96371882086167 - - type: cosine_recall - value: 65.11873350923483 - - type: dot_accuracy - value: 83.14358943792097 - - type: dot_accuracy_threshold - value: 82.20133781433105 - - type: dot_ap - value: 63.42166690978694 - - type: dot_f1 - value: 60.19512195121952 - - type: dot_f1_threshold - value: 78.19918394088745 - - type: dot_precision - value: 55.96371882086167 - - type: dot_recall - value: 65.11873350923483 - - type: euclidean_accuracy - value: 83.14358943792097 - - type: euclidean_accuracy_threshold - value: 59.66349244117737 - - type: euclidean_ap - value: 63.42169082838296 - - type: euclidean_f1 - value: 60.19512195121952 - - type: euclidean_f1_threshold - value: 66.03152751922607 - - type: euclidean_precision - value: 55.96371882086167 - - type: euclidean_recall - value: 65.11873350923483 - - type: main_score - value: 63.562161347378336 - - type: manhattan_accuracy - value: 83.17339214400667 - - type: manhattan_accuracy_threshold - value: 2131.719970703125 - - type: manhattan_ap - value: 63.562161347378336 - - type: manhattan_f1 - value: 60.42120551924474 - - type: manhattan_f1_threshold - value: 2370.509910583496 - - type: manhattan_precision - value: 55.81395348837209 - - type: manhattan_recall - value: 65.85751978891821 - - type: max_accuracy - value: 83.17339214400667 - - type: max_ap - value: 63.562161347378336 - - type: max_f1 - value: 60.42120551924474 - - type: max_precision - value: 55.96371882086167 - - type: max_recall - value: 65.85751978891821 - - type: similarity_accuracy - value: 83.14358943792097 - - type: similarity_accuracy_threshold - value: 82.20133781433105 - - type: similarity_ap - value: 63.42168849627343 - - type: similarity_f1 - value: 60.19512195121952 - - type: similarity_f1_threshold - value: 78.19918394088745 - - type: similarity_precision - value: 55.96371882086167 - - type: similarity_recall - value: 65.11873350923483 - task: - type: PairClassification - - dataset: - config: default - name: MTEB TwitterURLCorpus (default) - revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf - split: test - type: mteb/twitterurlcorpus-pairclassification - metrics: - - type: cosine_accuracy - value: 88.78798463150541 - - type: cosine_accuracy_threshold - value: 79.96333837509155 - - type: cosine_ap - value: 86.19745333095996 - - type: cosine_f1 - value: 77.74792925901053 - - type: cosine_f1_threshold - value: 77.78993844985962 - - type: cosine_precision - value: 75.42348342261474 - - type: cosine_recall - value: 80.2202032645519 - - type: dot_accuracy - value: 88.78798463150541 - - type: dot_accuracy_threshold - value: 79.96333241462708 - - type: dot_ap - value: 86.19745428025062 - - type: dot_f1 - value: 77.74792925901053 - - type: dot_f1_threshold - value: 77.78993844985962 - - type: dot_precision - value: 75.42348342261474 - - type: dot_recall - value: 80.2202032645519 - - type: euclidean_accuracy - value: 88.78798463150541 - - type: euclidean_accuracy_threshold - value: 63.303494453430176 - - type: euclidean_ap - value: 86.19745308448735 - - type: euclidean_f1 - value: 77.74792925901053 - - type: euclidean_f1_threshold - value: 66.64841771125793 - - type: euclidean_precision - value: 75.42348342261474 - - type: euclidean_recall - value: 80.2202032645519 - - type: main_score - value: 86.25813794250404 - - type: manhattan_accuracy - value: 88.79962743043428 - - type: manhattan_accuracy_threshold - value: 2239.702606201172 - - type: manhattan_ap - value: 86.25813794250404 - - type: manhattan_f1 - value: 77.84831272970264 - - type: manhattan_f1_threshold - value: 2393.1716918945312 - - type: manhattan_precision - value: 75.16667861495448 - - type: manhattan_recall - value: 80.72836464428703 - - type: max_accuracy - value: 88.79962743043428 - - type: max_ap - value: 86.25813794250404 - - type: max_f1 - value: 77.84831272970264 - - type: max_precision - value: 75.42348342261474 - - type: max_recall - value: 80.72836464428703 - - type: similarity_accuracy - value: 88.78798463150541 - - type: similarity_accuracy_threshold - value: 79.96333837509155 - - type: similarity_ap - value: 86.19745333095996 - - type: similarity_f1 - value: 77.74792925901053 - - type: similarity_f1_threshold - value: 77.78993844985962 - - type: similarity_precision - value: 75.42348342261474 - - type: similarity_recall - value: 80.2202032645519 - task: - type: PairClassification tags: -- mteb -license: mit -language: -- ru -- en -pipeline_tag: feature-extraction +- sentence-transformers +- sentence-similarity +- feature-extraction +pipeline_tag: sentence-similarity +library_name: sentence-transformers --- -## Giga-Embeddings-instruct -- Base Decoder-only LLM: GigaChat-3b -- Pooling Type: Latent-Attention -- Embedding Dimension: 2048 -## Использование +# SentenceTransformer -Ниже приведен пример кодирования запросов и текстов. +This is a [sentence-transformers](https://www.SBERT.net) model trained. It maps sentences & paragraphs to a 2048-dimensional dense vector space and can be used for semantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and more. +## Model Details -### Transformers +### Model Description +- **Model Type:** Sentence Transformer + +- **Maximum Sequence Length:** None tokens +- **Output Dimensionality:** 2048 dimensions +- **Similarity Function:** Cosine Similarity + + + -```python -import os -import torch -import torch.nn.functional as F -from transformers import AutoTokenizer, AutoModel +### Model Sources -# Each query needs to be accompanied by an corresponding instruction describing the task. -task_name_to_instruct = {"example": "Given a question, retrieve passages that answer the question",} +- **Documentation:** [Sentence Transformers Documentation](https://sbert.net) +- **Repository:** [Sentence Transformers on GitHub](https://github.com/UKPLab/sentence-transformers) +- **Hugging Face:** [Sentence Transformers on Hugging Face](https://huggingface.co/models?library=sentence-transformers) -query_prefix = task_name_to_instruct["example"] + "\nquestion: " -queries = [ - 'are judo throws allowed in wrestling?', - 'how to become a radiology technician in michigan?' -] +### Full Model Architecture -# No instruction needed for retrieval passages -passage_prefix = "" -passages = [ - "Since you're reading this, you are probably someone from a judo background or someone who is just wondering how judo techniques can be applied under wrestling rules. So without further ado, let's get to the question. Are Judo throws allowed in wrestling? Yes, judo throws are allowed in freestyle and folkstyle wrestling. You only need to be careful to follow the slam rules when executing judo throws. In wrestling, a slam is lifting and returning an opponent to the mat with unnecessary force.", - "Below are the basic steps to becoming a radiologic technologist in Michigan:Earn a high school diploma. As with most careers in health care, a high school education is the first step to finding entry-level employment. Taking classes in math and science, such as anatomy, biology, chemistry, physiology, and physics, can help prepare students for their college studies and future careers.Earn an associate degree. Entry-level radiologic positions typically require at least an Associate of Applied Science. Before enrolling in one of these degree programs, students should make sure it has been properly accredited by the Joint Review Committee on Education in Radiologic Technology (JRCERT).Get licensed or certified in the state of Michigan." -] +``` +SentenceTransformer( + (0): Transformer({'max_seq_length': None, 'do_lower_case': False}) with Transformer model: GigarEmbedModel + (1): Pooling({'word_embedding_dimension': 2048, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False, 'include_prompt': True}) +) +``` + +## Usage -# load model with tokenizer -model = AutoModel.from_pretrained('ai-sage/Giga-Embeddings-instruct', trust_remote_code=True) +### Direct Usage (Sentence Transformers) -# get the embeddings -query_embeddings = model.encode(queries, instruction=query_prefix) -passage_embeddings = model.encode(passages, instruction=passage_prefix) +First install the Sentence Transformers library: -# normalize embeddings -query_embeddings = F.normalize(query_embeddings, p=2, dim=1) -passage_embeddings = F.normalize(passage_embeddings, p=2, dim=1) +```bash +pip install -U sentence-transformers +``` -scores = (query_embeddings @ passage_embeddings.T) * 100 -print(scores.tolist()) +Then you can load this model and run inference. +```python +from sentence_transformers import SentenceTransformer + +# Download from the 🤗 Hub +model = SentenceTransformer("sentence_transformers_model_id") +# Run inference +sentences = [ + 'The weather is lovely today.', + "It's so sunny outside!", + 'He drove to the stadium.', +] +embeddings = model.encode(sentences) +print(embeddings.shape) +# [3, 2048] + +# Get the similarity scores for the embeddings +similarities = model.similarity(embeddings, embeddings) +print(similarities.shape) +# [3, 3] ``` -## Инструктивность + + + -**Примеры инструкций для симметричных задач:** -- `"Retrieve semantically similar text \ntext: {query}"` -- `"Given a text, retrieve semantically similar text \ntext: {query}"` -- `"Дано предложение, необходимо найти его парафраз \nпредложение: {query}"` -- `"Классифицируй отзыв на товар как положительный, отрицательный или нейтральный \nотзыв: {query}"` -- `"Классифицируй чувствительную тему по запросу \nзапрос: {query}"` + -Такой подход особенно эффективен для задач поиска и извлечения информации, таких как поиск релевантных документов или извлечение ответов из текста. + -Использование инструкций позволяет значительно улучшить качество поиска и релевантность результатов, что подтверждаетс�� тестами на бенчмарках, таких как RuBQ. Для симметричных задач добавление инструкции перед каждым запросом обеспечивает согласованность и повышает точность модели. + -Эта модель инициализирована pretrain моделью GigaChat и дополнительно обучена на смеси английских и русских данных. Однако, поскольку pretrain GigaChat'a делался в основном на русскоязычных данных, мы рекомендуем использовать эту модель только для русского языка. +## Training Details -## FAQ +### Framework Versions +- Python: 3.10.13 +- Sentence Transformers: 3.3.1 +- Transformers: 4.46.3 +- PyTorch: 2.1.1+cu121 +- Accelerate: 1.2.1 +- Datasets: 2.21.0 +- Tokenizers: 0.20.3 -1. Нужно ли добавлять инструкции к запросу? +## Citation -Да, именно так модель обучалась, иначе вы увидите снижение производительности. Определение задачи должно быть инструкцией в одном предложении, которая описывает задачу. Это способ настройки текстовых эмбеддингов для разных сценариев с помощью инструкций на естественном языке. +### BibTeX -С другой стороны, добавлять инструкции на сторону документа не требуется. + -Разные версии библиотек transformers и pytorch могут вызывать незначительные, но ненулевые различия в производительности. + -## Ограничения + \ No newline at end of file