Add new SparseEncoder model

README.md

@@ -8,28 +8,30 @@ tags:
 - sparse
 - splade
 - generated_from_trainer
-- dataset_size:100000
+- dataset_size:800000
 - loss:SpladeLoss
 - loss:SparseMultipleNegativesRankingLoss
 - loss:FlopsLoss
 base_model: prajjwal1/bert-tiny
 widget:
-- text: 'The cause of simple liver cysts isn''t known, but they may be the result
-    of a malformation present at birth. Rarely, liver cysts may indicate a serious,
-    underlying condition such as: 1  Polycystic liver disease, an inherited disorder.  Echinococcus
-    infection, a parasitic infection.'
-- text: 'Foods that have roughly the same number of calories from fats, calories,
-    and protein will be found closer to the center of the pyramid. Read more about
-    the Caloric Ratio Pyramid. Caloric Ratio Pyramid for Apples, raw, with skin [Includes
-    USDA commodity food A343] Nutrition Facts & Calories. - In Internet Explorer 7
-    you will need to adjust the default Shrink To Fit setting. Note: Printing via
-    Mac Firefox is currently not supported. - In Internet Explorer 7 you will need
-    to adjust the default Shrink To Fit setting.'
-- text: average age virginity loss
-- text: verb (used with object), subsidized, subsidizing. 1. to furnish or aid with
-    a subsidy. 2. to purchase the assistance of by the payment of a subsidy. 3. to
-    secure the cooperation of by bribery; buy over. Also, especially British, subsidise.
-- text: benefits of health care act
+- text: how much do private cleaners charge per hour
+- text: atlantic ocean air currents affects climate
+- text: RNA polymerase is the core enzyme in transcription which needs proteins known
+    as transcription factors to bind to the DNA promoter. also DNA plymerase... can't
+    remember what about it though. DNA polymerase isn't involved in DNA transcription.
+    However, DNA polymerase IS involved in DNA REPLICATION.
+- text: Exploit:JS/Axpergle.E Virus is a threatening Trojan horse which gets itself
+    loaded when you turn on your computer and eats up lots of system resources. Once
+    this Exploit:JS/Axpergle.E virus successfully enters your operating system, your
+    computer will be subjected to a variety of errors and drive you mad.) Exploit:JS/Axpergle.E
+    Virus corrupts the data and files saved on your computer hard drive terribly.
+    2) Exploit:JS/Axpergle.E Virus changes the registry entry to get itself launched
+    at system startup.
+- text: --No depreciation deduction shall be allowed under this section (and no depreciation
+    or amortization deduction shall be allowed under any other provision of this subtitle)
+    to the taxpayer for any term interest in property for any period during which
+    the remainder interest in such property is held (directly or indirectly) by a
+    related person.
 pipeline_tag: feature-extraction
 library_name: sentence-transformers
 metrics:
@@ -63,61 +65,61 @@ model-index:
       type: unknown
     metrics:
     - type: dot_accuracy@1
-      value: 0.7258
+      value: 0.457
       name: Dot Accuracy@1
     - type: dot_accuracy@3
-      value: 0.8584
+      value: 0.7572
       name: Dot Accuracy@3
     - type: dot_accuracy@5
-      value: 0.8986
+      value: 0.8574
       name: Dot Accuracy@5
     - type: dot_accuracy@10
-      value: 0.9372
+      value: 0.929
       name: Dot Accuracy@10
     - type: dot_precision@1
-      value: 0.7258
+      value: 0.457
       name: Dot Precision@1
     - type: dot_precision@3
-      value: 0.2861333333333333
+      value: 0.25906666666666667
       name: Dot Precision@3
     - type: dot_precision@5
-      value: 0.17972
+      value: 0.178
       name: Dot Precision@5
     - type: dot_precision@10
-      value: 0.09371999999999998
+      value: 0.09714
       name: Dot Precision@10
     - type: dot_recall@1
-      value: 0.7258
+      value: 0.44155
       name: Dot Recall@1
     - type: dot_recall@3
-      value: 0.8584
+      value: 0.7427833333333334
       name: Dot Recall@3
     - type: dot_recall@5
-      value: 0.8986
+      value: 0.8471666666666666
       name: Dot Recall@5
     - type: dot_recall@10
-      value: 0.9372
+      value: 0.9223
       name: Dot Recall@10
     - type: dot_ndcg@10
-      value: 0.8335744020505143
+      value: 0.6931598312411338
       name: Dot Ndcg@10
     - type: dot_mrr@10
-      value: 0.8001331746031765
+      value: 0.6234866666666686
       name: Dot Mrr@10
     - type: dot_map@100
-      value: 0.8025817896431575
+      value: 0.6191148055389254
       name: Dot Map@100
     - type: query_active_dims
-      value: 34.04159927368164
+      value: 21.215999603271484
       name: Query Active Dims
     - type: query_sparsity_ratio
-      value: 0.9988846864794678
+      value: 0.9993048948429568
       name: Query Sparsity Ratio
     - type: corpus_active_dims
-      value: 192.54740061035156
+      value: 159.5419082486014
       name: Corpus Active Dims
     - type: corpus_sparsity_ratio
-      value: 0.9936915208501949
+      value: 0.99477288813811
       name: Corpus Sparsity Ratio
 ---
 
@@ -167,15 +169,15 @@ Then you can load this model and run inference.
 from sentence_transformers import SparseEncoder
 
 # Download from the 🤗 Hub
-model = SparseEncoder("rasyosef/SPLADE-BERT-Tiny")
+model = SparseEncoder("yosefw/SPLADE-BERT-Tiny-v2")
 # Run inference
 queries = [
-    "benefits of health care act",
+    "what code section is depreciation",
 ]
 documents = [
-    'Every health insurance plan sold in the Marketplace will offer 10 essential health benefits. These essential health benefits include at least the following items and services: 1  Outpatient care—the kind you get without being admitted to a hospital.  Trips to the emergency room.',
-    'Essential Health Benefits. A set of 10 categories of services health insurance plans must cover under the Affordable Care Act. These include doctors’ services, inpatient and outpatient hospital care, prescription drug coverage, pregnancy and childbirth, mental health services, and more. Some plans cover more services.',
-    'Find Baker Hughes in Minot, ND 58701-5870 on Yellowbook. Get contact details or leave a review about this business.',
+    'Section 179 depreciation deduction. Section 179 of the United States Internal Revenue Code (26 U.S.C. § 179), allows a taxpayer to elect to deduct the cost of certain types of property on their income taxes as an expense, rather than requiring the cost of the property to be capitalized and depreciated.',
+    '--No depreciation deduction shall be allowed under this section (and no depreciation or amortization deduction shall be allowed under any other provision of this subtitle) to the taxpayer for any term interest in property for any period during which the remainder interest in such property is held (directly or indirectly) by a related person.',
+    'Depreciation - Amortization Code. Refer to the IRS Instructions for Form 4562, Line 42, for the amortization code.',
 ]
 query_embeddings = model.encode_query(queries)
 document_embeddings = model.encode_document(documents)
@@ -185,7 +187,7 @@ print(query_embeddings.shape, document_embeddings.shape)
 # Get the similarity scores for the embeddings
 similarities = model.similarity(query_embeddings, document_embeddings)
 print(similarities)
-# tensor([[13.6736, 18.2270,  0.0508]])
+# tensor([[17.0167, 11.4943, 13.8083]])
 ```
 
 <!--
@@ -222,25 +224,25 @@ You can finetune this model on your own dataset.
 
 | Metric                | Value      |
 |:----------------------|:-----------|
-| dot_accuracy@1        | 0.7258     |
-| dot_accuracy@3        | 0.8584     |
-| dot_accuracy@5        | 0.8986     |
-| dot_accuracy@10       | 0.9372     |
-| dot_precision@1       | 0.7258     |
-| dot_precision@3       | 0.2861     |
-| dot_precision@5       | 0.1797     |
-| dot_precision@10      | 0.0937     |
-| dot_recall@1          | 0.7258     |
-| dot_recall@3          | 0.8584     |
-| dot_recall@5          | 0.8986     |
-| dot_recall@10         | 0.9372     |
-| **dot_ndcg@10**       | **0.8336** |
-| dot_mrr@10            | 0.8001     |
-| dot_map@100           | 0.8026     |
-| query_active_dims     | 34.0416    |
-| query_sparsity_ratio  | 0.9989     |
-| corpus_active_dims    | 192.5474   |
-| corpus_sparsity_ratio | 0.9937     |
+| dot_accuracy@1        | 0.457      |
+| dot_accuracy@3        | 0.7572     |
+| dot_accuracy@5        | 0.8574     |
+| dot_accuracy@10       | 0.929      |
+| dot_precision@1       | 0.457      |
+| dot_precision@3       | 0.2591     |
+| dot_precision@5       | 0.178      |
+| dot_precision@10      | 0.0971     |
+| dot_recall@1          | 0.4415     |
+| dot_recall@3          | 0.7428     |
+| dot_recall@5          | 0.8472     |
+| dot_recall@10         | 0.9223     |
+| **dot_ndcg@10**       | **0.6932** |
+| dot_mrr@10            | 0.6235     |
+| dot_map@100           | 0.6191     |
+| query_active_dims     | 21.216     |
+| query_sparsity_ratio  | 0.9993     |
+| corpus_active_dims    | 159.5419   |
+| corpus_sparsity_ratio | 0.9948     |
 
 <!--
 ## Bias, Risks and Limitations
@@ -260,25 +262,25 @@ You can finetune this model on your own dataset.
 
 #### Unnamed Dataset
 
-* Size: 100,000 training samples
-* Columns: <code>query</code>, <code>positive</code>, and <code>negative</code>
+* Size: 800,000 training samples
+* Columns: <code>query</code>, <code>positive</code>, <code>negative_1</code>, and <code>negative_2</code>
 * Approximate statistics based on the first 1000 samples:
-  |         | query                                                                            | positive                                                                            | negative                                                                            |
-  |:--------|:---------------------------------------------------------------------------------|:------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------|
-  | type    | string                                                                           | string                                                                              | string                                                                              |
-  | details | <ul><li>min: 4 tokens</li><li>mean: 8.07 tokens</li><li>max: 21 tokens</li></ul> | <ul><li>min: 21 tokens</li><li>mean: 81.21 tokens</li><li>max: 236 tokens</li></ul> | <ul><li>min: 20 tokens</li><li>mean: 79.54 tokens</li><li>max: 187 tokens</li></ul> |
+  |         | query                                                                            | positive                                                                            | negative_1                                                                          | negative_2                                                                          |
+  |:--------|:---------------------------------------------------------------------------------|:------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------|
+  | type    | string                                                                           | string                                                                              | string                                                                              | string                                                                              |
+  | details | <ul><li>min: 4 tokens</li><li>mean: 9.03 tokens</li><li>max: 30 tokens</li></ul> | <ul><li>min: 15 tokens</li><li>mean: 81.92 tokens</li><li>max: 220 tokens</li></ul> | <ul><li>min: 22 tokens</li><li>mean: 78.63 tokens</li><li>max: 227 tokens</li></ul> | <ul><li>min: 18 tokens</li><li>mean: 78.11 tokens</li><li>max: 236 tokens</li></ul> |
 * Samples:
-  | query                                               | positive                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | negative                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
-  |:----------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-  | <code>can a spouse make you move out</code>         | <code>If you and your spouse purchased a home together or the house was bought during the course of the marriage, you likely cannot make a spouse move out during divorce.However, if your spouse is engaging in abusive behavior towards you or towards your children, then you could seek a protective order or an injunction.his would be the case if you purchased the home before marriage; if the home is in your name only and if your spouse has made no contributions to the home during the time you were husband and wife.</code>               | <code>Many courts will not award a final restraining order unless there’s been actual physical violence. If you get a temporary restraining order because your spouse threatened violence, it may not be enough to keep him out of the home permanently, pending your divorce.tep 1. Talk it out. Your spouse might not want to be in the house with you any more than you want to live there with him, but he may have some viable concerns about moving out. Try to come up with a plan to address those concerns.</code> |
-  | <code>can i get pregnant right before period</code> | <code>simran47. It is difficult to get pregnant right before your period, since ovulation takes place around 2 weeks before the next period. This is true among women who have regular menstrual cycles and ovulation can be predicted to occur around mid-cycle. However, some women have irregular cycles. If your ovulation takes place later than expected (mid-cycle), say on the 24th day of the cycle instead of around 14th day, and fertilization and implantation take place, then pregnancy can occur right before your expected period.</code> | <code>The question about whether you can get pregnant on your period or not is probably the most common of the three situations noted above. The answer is yes and no. The probability is that you would not get pregnant having sex during your period.</code>                                                                                                                                                                                                                                                             |
-  | <code>average training mileage rates</code>         | <code>Beginning on Jan. 1, 2014, the standard mileage rates for the use of a car (also vans, pickups or panel trucks) will be: 1  56 cents per mile for business miles driven.  23.5 cents per mile driven for medical or moving purposes.</code>                                                                                                                                                                                                                                                                                                          | <code>Schneider Increases Mileage Rates. Trucker to Trucker is the major online resource for sourcing trucks for sale and for selling all trucking equipment, including rigs and semi trailers. Schneider National is a name known to all truckers and operators and has been in business for over three-quarters of a century.</code>                                                                                                                                                                                      |
+  | query                                                  | positive                                                                                                                                                                                                                                                                                                                                      | negative_1                                                                                                                                                                                                                                                                                                                         | negative_2                                                                                                                                                                                                                                                                                                |
+  |:-------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+  | <code>definition of vas deferens</code>                | <code>Vas deferens: The tube that connects the testes with the urethra. The vas deferens is a coiled duct that conveys sperm from the epididymis to the ejaculatory duct and the urethra.</code>                                                                                                                                              | <code>For further discussion of the vas deferens within the context of the structures and functions of reproduction and sexuality, please see the overview section “The Reproductive System.”. See also FERTILITY; TESTICLES; VASECTOMY.</code>                                                                                    | <code>1 Testicular cancer symptoms include a painless lump or swelling in a testicle, testicle or scrotum pain, a dull ache in the abdomen, back, or groin, and. 2  Urinary Tract Infections (UTIs)  A urinary tract infection (UTI) is an infection of the bladder, kidneys, ureters, or urethra.</code> |
+  | <code>how old is kieron williamson</code>              | <code>Kieron Williamson – the latest artist to be part of GoGoDragons! April 21, 2015. A 12-year-old artist, nicknamed Mini-Monet, is to unveil a sculpture of a dragon he has painted for GoGoDragons. Kieron Williamson, from Norfolk, who has so far earned about £2m, painted the 5ft-tall (1.5m) dragon for the event in Norwich.</code> | <code>8-year-old artist: Don't call me Monet. London, England (CNN) -- He has the deft brush strokes of a seasoned artist, but Kieron Williamson is just eight years old. The boy from Norfolk, in eastern England, has been hailed by the British press as a mini Monet, a reference to the famous French impressionist.</code>   | <code>Needless to say, this site does not tell you much about his football career (yet!), but the website will tell you everything there is to know about Kieron Williamson’s passion for oil, watercolour and pastel,</code>                                                                             |
+  | <code>when do you start showing third pregnancy</code> | <code>Yes | No Thank you! I am pregnant with my third child and I am definitly showing at 10 weeks. I am starting to wear some maternity clothes. My low low rise pre-pregnancy jeans still work. My biggest problem is shirts, but fortunately the style right now is loose shirts that look maternity.</code>                               | <code>Some women do not start to show until they are well into their second trimester or even the start of their third trimester. If you are overweight at the start of your pregnancy, you may not gain as much weight during your pregnancy and may not begin to show until later into your pregnancy. Average: 3.591215.</code> | <code>There isn't a set time when moms-to-be start sporting an obviously pregnant belly – every woman is different. Some women keep their pre-pregnancy belly far into the second trimester, while others start showing in the first trimester.</code>                                                    |
 * Loss: [<code>SpladeLoss</code>](https://sbert.net/docs/package_reference/sparse_encoder/losses.html#spladeloss) with these parameters:
   ```json
   {
       "loss": "SparseMultipleNegativesRankingLoss(scale=1.0, similarity_fct='dot_score')",
-      "document_regularizer_weight": 0.001,
-      "query_regularizer_weight": 0.002
+      "document_regularizer_weight": 0.003,
+      "query_regularizer_weight": 0.005
   }
   ```
 
@@ -286,14 +288,17 @@ You can finetune this model on your own dataset.
 #### Non-Default Hyperparameters
 
 - `eval_strategy`: epoch
-- `per_device_train_batch_size`: 32
-- `per_device_eval_batch_size`: 32
-- `learning_rate`: 4e-05
-- `num_train_epochs`: 4
+- `per_device_train_batch_size`: 16
+- `per_device_eval_batch_size`: 16
+- `gradient_accumulation_steps`: 4
+- `learning_rate`: 6e-05
+- `num_train_epochs`: 6
 - `lr_scheduler_type`: cosine
 - `warmup_ratio`: 0.025
 - `fp16`: True
+- `load_best_model_at_end`: True
 - `optim`: adamw_torch_fused
+- `push_to_hub`: True
 - `batch_sampler`: no_duplicates
 
 #### All Hyperparameters
@@ -303,20 +308,20 @@ You can finetune this model on your own dataset.
 - `do_predict`: False
 - `eval_strategy`: epoch
 - `prediction_loss_only`: True
-- `per_device_train_batch_size`: 32
-- `per_device_eval_batch_size`: 32
+- `per_device_train_batch_size`: 16
+- `per_device_eval_batch_size`: 16
 - `per_gpu_train_batch_size`: None
 - `per_gpu_eval_batch_size`: None
-- `gradient_accumulation_steps`: 1
+- `gradient_accumulation_steps`: 4
 - `eval_accumulation_steps`: None
 - `torch_empty_cache_steps`: None
-- `learning_rate`: 4e-05
+- `learning_rate`: 6e-05
 - `weight_decay`: 0.0
 - `adam_beta1`: 0.9
 - `adam_beta2`: 0.999
 - `adam_epsilon`: 1e-08
 - `max_grad_norm`: 1.0
-- `num_train_epochs`: 4
+- `num_train_epochs`: 6
 - `max_steps`: -1
 - `lr_scheduler_type`: cosine
 - `lr_scheduler_kwargs`: {}
@@ -356,7 +361,7 @@ You can finetune this model on your own dataset.
 - `disable_tqdm`: False
 - `remove_unused_columns`: True
 - `label_names`: None
-- `load_best_model_at_end`: False
+- `load_best_model_at_end`: True
 - `ignore_data_skip`: False
 - `fsdp`: []
 - `fsdp_min_num_params`: 0
@@ -377,7 +382,7 @@ You can finetune this model on your own dataset.
 - `dataloader_persistent_workers`: False
 - `skip_memory_metrics`: True
 - `use_legacy_prediction_loop`: False
-- `push_to_hub`: False
+- `push_to_hub`: True
 - `resume_from_checkpoint`: None
 - `hub_model_id`: None
 - `hub_strategy`: every_save
@@ -420,23 +425,25 @@ You can finetune this model on your own dataset.
 </details>
 
 ### Training Logs
-| Epoch | Step  | Training Loss | dot_ndcg@10 |
-|:-----:|:-----:|:-------------:|:-----------:|
-| 1.0   | 3125  | 22.6481       | 0.7960      |
-| 2.0   | 6250  | 0.6743        | 0.8296      |
-| 3.0   | 9375  | 0.5636        | 0.8333      |
-| 4.0   | 12500 | 0.5257        | 0.8336      |
-| -1    | -1    | -             | 0.8336      |
+| Epoch   | Step      | Training Loss | dot_ndcg@10 |
+|:-------:|:---------:|:-------------:|:-----------:|
+| 1.0     | 12500     | 11.5771       | 0.6587      |
+| 2.0     | 25000     | 0.7888        | 0.6810      |
+| 3.0     | 37500     | 0.7271        | 0.6884      |
+| 4.0     | 50000     | 0.6774        | 0.6920      |
+| 5.0     | 62500     | 0.6436        | 0.6912      |
+| **6.0** | **75000** | **0.6274**    | **0.6932**  |
 
+* The bold row denotes the saved checkpoint.
 
 ### Framework Versions
-- Python: 3.11.13
+- Python: 3.11.11
 - Sentence Transformers: 5.0.0
-- Transformers: 4.53.0
+- Transformers: 4.53.1
 - PyTorch: 2.6.0+cu124
-- Accelerate: 1.8.1
+- Accelerate: 1.5.2
 - Datasets: 3.6.0
-- Tokenizers: 0.21.2
+- Tokenizers: 0.21.1
 
 ## Citation
 

config.json

@@ -17,7 +17,7 @@
   "pad_token_id": 0,
   "position_embedding_type": "absolute",
   "torch_dtype": "float32",
-  "transformers_version": "4.53.0",
+  "transformers_version": "4.53.1",
   "type_vocab_size": 2,
   "use_cache": true,
   "vocab_size": 30522

config_sentence_transformers.json

@@ -2,7 +2,7 @@
   "model_type": "SparseEncoder",
   "__version__": {
     "sentence_transformers": "5.0.0",
-    "transformers": "4.53.0",
+    "transformers": "4.53.1",
     "pytorch": "2.6.0+cu124"
   },
   "prompts": {

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:90fd50282273fc3ba0af07f264845f68fc9d5e0c23813961674d30b94db33f12
+oid sha256:6b234281fbc722918304ccf3e74f1b216cc2deb2c3197034a6bf90f2a6b7e3fe
 size 17671560

special_tokens_map.json

@@ -1,7 +1,37 @@
 {
-  "cls_token": "[CLS]",
-  "mask_token": "[MASK]",
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "unk_token": "[UNK]"
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer_config.json

@@ -47,12 +47,19 @@
   "do_lower_case": true,
   "extra_special_tokens": {},
   "mask_token": "[MASK]",
-  "model_max_length": 1000000000000000019884624838656,
+  "max_length": 512,
+  "model_max_length": 512,
   "never_split": null,
+  "pad_to_multiple_of": null,
   "pad_token": "[PAD]",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
   "sep_token": "[SEP]",
+  "stride": 0,
   "strip_accents": null,
   "tokenize_chinese_chars": true,
   "tokenizer_class": "BertTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "[UNK]"
 }