Upload folder using huggingface_hub

1_Pooling/config.json

@@ -1,7 +1,7 @@
 {
-  "word_embedding_dimension": 768,
-  "pooling_mode_cls_token": false,
-  "pooling_mode_mean_tokens": true,
+  "word_embedding_dimension": 384,
+  "pooling_mode_cls_token": true,
+  "pooling_mode_mean_tokens": false,
   "pooling_mode_max_tokens": false,
   "pooling_mode_mean_sqrt_len_tokens": false,
   "pooling_mode_weightedmean_tokens": false,

README.md

@@ -4,72 +4,63 @@ tags:
 - sentence-similarity
 - feature-extraction
 - generated_from_trainer
-- dataset_size:1788
-- loss:MultipleNegativesRankingLoss
-base_model: sentence-transformers/all-mpnet-base-v2
+- dataset_size:1798
+- loss:TripletLoss
+base_model: BAAI/bge-small-en-v1.5
 widget:
-- source_sentence: How many holdings do [TICKER1] and [TICKER2] have in common?
+- source_sentence: How will the NIKKEI 225 affect my portfolio
   sentences:
-  - '[{"get_portfolio(None,True,None)": "portfolio"}, {"factor_contribution(''portfolio'',''<DATES>'',''sector'',''<A_SECTOR>'',''returns'')":
-    "portfolio"}]'
-  - '[{"get_portfolio(None,True,None)": "portfolio"}, {"factor_contribution(''portfolio'',''<DATES>'',''asset_class'',''<AN_ASSET_TYPE>'',''portfolio'')":
+  - '[{"get_portfolio(None,True,None)": "portfolio"}, {"stress_test(''portfolio'',''nikkei_225'',None,''up'')":
+    "stress_test"}]'
+  - '[{"get_portfolio(None,True,None)": "portfolio"}, {"get_attribute(''portfolio'',[''dividend
+    yield''],''<DATES>'')": "portfolio"}, {"calculate(''portfolio'',[''dividend yield'',
+    ''marketValue''],''multiply'',''div_income'')": "portfolio"}, {"sort(''portfolio'',''div_income'',''desc'')":
     "portfolio"}]'
-  - '[{"compare([[''<TICKER1>'', ''<TICKER2>'']], None, ''fund_holdings'')": "comparison_data"}]'
-- source_sentence: best performing MFs in my portfolio
+  - '[{"get_portfolio(None,True,None)": "portfolio"}, {"stress_test(''portfolio'',''nikkei_225'',None,None)":
+    "stress_test"}]'
+- source_sentence: What’s the [DATES] trend of the [A_SECTOR] sector
   sentences:
-  - '[{"get_portfolio(None,True,None)": "portfolio"}, {"factor_contribution(''portfolio'',''<DATES>'',''asset_class'',''<AN_ASSET_TYPE>'',''portfolio'')":
-    "portfolio"}]'
-  - '[{"get_portfolio([''type''],True,None)": "portfolio"}, {"filter(''portfolio'',''type'',''=='',''MF'')":
-    "portfolio"}, {"get_attribute(''portfolio'',[''gains''],''<DATES>'')": "portfolio"},
-    {"filter(''portfolio'',''gains'',''>'',''0'')": "portfolio"}, {"sort(''portfolio'',''gains'',''desc'')":
-    "portfolio"}]'
-  - '[{"get_dividend_history([''<TICKER>''],None)": "<TICKER>_dividend_history"}]'
-- source_sentence: How will the Cotation Assistée en Continu affect my portfolio?
+  - '[{"get_portfolio(None,True,None)": "portfolio"}, {"get_attribute(''portfolio'',[''<A_THEME>'',
+    ''risk''],''<DATES>'')": "portfolio"}, {"filter(''portfolio'',''<A_THEME>'',''>'',''0.01'')":
+    "portfolio"}, {"sort(''portfolio'',''risk'',''asc'')": "portfolio"}]'
+  - '[{"get_attribute([''<A_SECTOR>''],[''returns''],''<DATES>'')":"sector_returns"}]'
+  - '[{"get_news_articles(None,None,[''<A_SECTOR>''],''<DATES>'')": "news_data"}]'
+- source_sentence: How will rising gold commodities affect my portfolio
   sentences:
-  - '[{"get_portfolio(None,True,None)": "portfolio"}, {"get_expected_attribute(''portfolio'',[''volatility''])":
-    "portfolio"}, {"sort(''portfolio'',''volatility'',''asc'')": "portfolio"}]'
-  - '[{"get_portfolio(None,True,None)": "portfolio"}, {"stress_test(''portfolio'',''cac_40'',None,''up'')":
+  - '[{"get_attribute([''<TICKER>''],[''returns''],''<DATES>'')":"<TICKER>_returns"}]'
+  - '[{"get_portfolio(None,True,None)": "portfolio"}, {"stress_test(''portfolio'',''gold'',None,None)":
     "stress_test"}]'
-  - '[{"get_portfolio(None,True,None)": "portfolio"}, {"stress_test(''portfolio'',''cac_40'',None,None)":
+  - '[{"get_portfolio(None,True,None)": "portfolio"}, {"stress_test(''portfolio'',''gold'',None,''up'')":
     "stress_test"}]'
-- source_sentence: what were the latest earnings of [TICKER]
+- source_sentence: what percent of my account is in [AN_ASSET_TYPE]
   sentences:
-  - '[{"get_earnings_announcements([''<TICKER>''],''last quarter'')": "<TICKER>_earnings"}]'
-  - '[{"newsletter_search([''<NEWSLETTER_NAME>''],None,None,''<DATES>'',True)": "newsletter_chunks"}]'
-  - '[{"get_portfolio([''type''],True,None)": "portfolio"}, {"get_attribute(''portfolio'',[''risk''],''<DATES>'')":
-    "portfolio"}, {"sort(''portfolio'',''risk'',''desc'')": "portfolio"}]'
-- source_sentence: what would be my returns if I didn't own [TICKER]
+  - '[{"get_portfolio(None,True,None)": "portfolio"}, {"factor_contribution(''portfolio'',''<DATES>'',''asset_class'',''<AN_ASSET_TYPE>'',''portfolio'')":
+    "portfolio"}]'
+  - '[{"get_news_articles(None,None,[''<A_SECTOR>''],''<DATES>'')": "news_data"}]'
+  - '[{"get_attribute([''<TICKER>''],[''<AN_ASSET_TYPE>''],''<DATES>'')":"<TICKER>_data"}]'
+- source_sentence: Can I get a performance check-in
   sentences:
-  - '[{"get_portfolio(None,True,None)": "portfolio"}, {"newsletter_search([''<NEWSLETTER_NAME>''],''portfolio'',None,None,False)":
-    "newsletter_chunks"}]'
-  - '[{"newsletter_search(None,[''<TICKER>''],''query'',None,False)": "newsletter_chunks"},
-    {"get_portfolio([''weight''],True,None)": "portfolio"}, {"get_attribute(''portfolio'',[''<TICKER>''],None)":
-    "portfolio"}, {"calculate(''portfolio'',[''weight'', ''<TICKER>''],''multiply'',''expo_percentage_<TICKER>'')":
-    "portfolio"}, {"sort(''portfolio'',''expo_percentage_<TICKER>'',''desc'')": "portfolio"},
-    {"aggregate(''portfolio'',''ticker'',''expo_percentage_<TICKER>'',''sum'',None)":
-    "total_expo_percentage_<TICKER>"},{"analyze_impact(''portfolio'',''<TICKER>'',''sell'')":
-    "<TICKER>_sell_impact"}]'
-  - '[{"get_portfolio([''marketValue''],True,None)": "portfolio"}, {"get_attribute(''portfolio'',[''<TICKER1>'',''<TICKER2>''],''<DATES>'')":
-    "portfolio"}, {"calculate(''portfolio'',[''marketValue'', ''<TICKER1>''],''multiply'',''expo_<TICKER1>'')":
-    "portfolio"}, {"calculate(''portfolio'',[''marketValue'', ''<TICKER2>''],''multiply'',''expo_<TICKER2>'')":
-    "portfolio"}, {"sort(''portfolio'',''expo_<TICKER1>'',''desc'')": "portfolio"},
-    {"aggregate(''portfolio'',''ticker'',''expo_<TICKER1>'',''sum'',None)": "port_expo_<TICKER1>"},
-    {"aggregate(''portfolio'',''ticker'',''expo_<TICKER2>'',''sum'',None)": "port_expo_<TICKER2>"}]'
+  - '[{"search(''query'', ''match_type'', ''<TICKER>'')": "search_results"},{"compare([[''<TICKER>''],''search_results''],
+    [''yield''], None)": "comparison_data"}]'
+  - '[{"get_portfolio(None, True, None)": "portfolio"}, {"get_attribute(''portfolio'',[''gains''],''<DATES>'')":
+    "portfolio"}, {"sort(''portfolio'',''gains'',''desc'')": "portfolio"}]'
+  - '[{"get_portfolio(None,True,None)": "portfolio"},{"factor_contribution(''portfolio'',''<DATES>'',''security'',''<TICKER>'',''returns'')}":
+    "portfolio"}, {"get_attribute([''<TICKER>''],[''returns''],''<DATES>'')": "returns_<TICKER>"}]'
 pipeline_tag: sentence-similarity
 library_name: sentence-transformers
 ---
 
-# SentenceTransformer based on sentence-transformers/all-mpnet-base-v2
+# SentenceTransformer based on BAAI/bge-small-en-v1.5
 
-This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [sentence-transformers/all-mpnet-base-v2](https://huggingface.co/sentence-transformers/all-mpnet-base-v2). It maps sentences & paragraphs to a 768-dimensional dense vector space and can be used for semantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and more.
+This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [BAAI/bge-small-en-v1.5](https://huggingface.co/BAAI/bge-small-en-v1.5). It maps sentences & paragraphs to a 384-dimensional dense vector space and can be used for semantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and more.
 
 ## Model Details
 
 ### Model Description
 - **Model Type:** Sentence Transformer
-- **Base model:** [sentence-transformers/all-mpnet-base-v2](https://huggingface.co/sentence-transformers/all-mpnet-base-v2) <!-- at revision 12e86a3c702fc3c50205a8db88f0ec7c0b6b94a0 -->
-- **Maximum Sequence Length:** 384 tokens
-- **Output Dimensionality:** 768 dimensions
+- **Base model:** [BAAI/bge-small-en-v1.5](https://huggingface.co/BAAI/bge-small-en-v1.5) <!-- at revision 5c38ec7c405ec4b44b94cc5a9bb96e735b38267a -->
+- **Maximum Sequence Length:** 512 tokens
+- **Output Dimensionality:** 384 dimensions
 - **Similarity Function:** Cosine Similarity
 <!-- - **Training Dataset:** Unknown -->
 <!-- - **Language:** Unknown -->
@@ -85,8 +76,8 @@ This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [s
 
 ```
 SentenceTransformer(
-  (0): Transformer({'max_seq_length': 384, 'do_lower_case': False}) with Transformer model: MPNetModel 
-  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False, 'include_prompt': True})
+  (0): Transformer({'max_seq_length': 512, 'do_lower_case': True}) with Transformer model: BertModel 
+  (1): Pooling({'word_embedding_dimension': 384, 'pooling_mode_cls_token': True, 'pooling_mode_mean_tokens': False, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False, 'include_prompt': True})
   (2): Normalize()
 )
 ```
@@ -109,13 +100,13 @@ from sentence_transformers import SentenceTransformer
 model = SentenceTransformer("sentence_transformers_model_id")
 # Run inference
 sentences = [
-    "what would be my returns if I didn't own [TICKER]",
-    '[{"newsletter_search(None,[\'<TICKER>\'],\'query\',None,False)": "newsletter_chunks"}, {"get_portfolio([\'weight\'],True,None)": "portfolio"}, {"get_attribute(\'portfolio\',[\'<TICKER>\'],None)": "portfolio"}, {"calculate(\'portfolio\',[\'weight\', \'<TICKER>\'],\'multiply\',\'expo_percentage_<TICKER>\')": "portfolio"}, {"sort(\'portfolio\',\'expo_percentage_<TICKER>\',\'desc\')": "portfolio"}, {"aggregate(\'portfolio\',\'ticker\',\'expo_percentage_<TICKER>\',\'sum\',None)": "total_expo_percentage_<TICKER>"},{"analyze_impact(\'portfolio\',\'<TICKER>\',\'sell\')": "<TICKER>_sell_impact"}]',
-    '[{"get_portfolio([\'marketValue\'],True,None)": "portfolio"}, {"get_attribute(\'portfolio\',[\'<TICKER1>\',\'<TICKER2>\'],\'<DATES>\')": "portfolio"}, {"calculate(\'portfolio\',[\'marketValue\', \'<TICKER1>\'],\'multiply\',\'expo_<TICKER1>\')": "portfolio"}, {"calculate(\'portfolio\',[\'marketValue\', \'<TICKER2>\'],\'multiply\',\'expo_<TICKER2>\')": "portfolio"}, {"sort(\'portfolio\',\'expo_<TICKER1>\',\'desc\')": "portfolio"}, {"aggregate(\'portfolio\',\'ticker\',\'expo_<TICKER1>\',\'sum\',None)": "port_expo_<TICKER1>"}, {"aggregate(\'portfolio\',\'ticker\',\'expo_<TICKER2>\',\'sum\',None)": "port_expo_<TICKER2>"}]',
+    'Can I get a performance check-in',
+    '[{"get_portfolio(None, True, None)": "portfolio"}, {"get_attribute(\'portfolio\',[\'gains\'],\'<DATES>\')": "portfolio"}, {"sort(\'portfolio\',\'gains\',\'desc\')": "portfolio"}]',
+    '[{"get_portfolio(None,True,None)": "portfolio"},{"factor_contribution(\'portfolio\',\'<DATES>\',\'security\',\'<TICKER>\',\'returns\')}": "portfolio"}, {"get_attribute([\'<TICKER>\'],[\'returns\'],\'<DATES>\')": "returns_<TICKER>"}]',
 ]
 embeddings = model.encode(sentences)
 print(embeddings.shape)
-# [3, 768]
+# [3, 384]
 
 # Get the similarity scores for the embeddings
 similarities = model.similarity(embeddings, embeddings)
@@ -165,24 +156,24 @@ You can finetune this model on your own dataset.
 
 #### Unnamed Dataset
 
-* Size: 1,788 training samples
-* Columns: <code>sentence_0</code> and <code>sentence_1</code>
+* Size: 1,798 training samples
+* Columns: <code>sentence_0</code>, <code>sentence_1</code>, and <code>sentence_2</code>
 * Approximate statistics based on the first 1000 samples:
-  |         | sentence_0                                                                        | sentence_1                                                                          |
-  |:--------|:----------------------------------------------------------------------------------|:------------------------------------------------------------------------------------|
-  | type    | string                                                                            | string                                                                              |
-  | details | <ul><li>min: 4 tokens</li><li>mean: 12.78 tokens</li><li>max: 35 tokens</li></ul> | <ul><li>min: 20 tokens</li><li>mean: 75.42 tokens</li><li>max: 279 tokens</li></ul> |
+  |         | sentence_0                                                                        | sentence_1                                                                          | sentence_2                                                                          |
+  |:--------|:----------------------------------------------------------------------------------|:------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------|
+  | type    | string                                                                            | string                                                                              | string                                                                              |
+  | details | <ul><li>min: 4 tokens</li><li>mean: 12.37 tokens</li><li>max: 32 tokens</li></ul> | <ul><li>min: 20 tokens</li><li>mean: 71.59 tokens</li><li>max: 206 tokens</li></ul> | <ul><li>min: 20 tokens</li><li>mean: 73.42 tokens</li><li>max: 229 tokens</li></ul> |
 * Samples:
-  | sentence_0                                                                | sentence_1                                                                                                                                                                                                                                                                                           |
-  |:--------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-  | <code>What is the impact of [TICKER] on volatility in my portfolio</code> | <code>[{"get_portfolio(None,True,None)": "portfolio"}, {"factor_contribution('portfolio','<DATES>','volatility',None,'portfolio')": "portfolio"}]</code>                                                                                                                                             |
-  | <code>What is the [ATTRIBUTE] of [TICKER]</code>                          | <code>[{"get_attribute(['<TICKER>'],['<ATTRIBUTE>'],None)":"<TICKER>_data"}]</code>                                                                                                                                                                                                                  |
-  | <code>which of my mutual funds performed poorly [DATES]?</code>           | <code>[{"get_portfolio(['type'],True,None)": "portfolio"}, {"filter('portfolio','type','==','MF')": "portfolio"}, {"get_attribute('portfolio',['losses'],'<DATES>')": "portfolio"}, {"filter('portfolio','losses','<','0')": "portfolio"}, {"sort('portfolio','losses','asc')": "portfolio"}]</code> |
-* Loss: [<code>MultipleNegativesRankingLoss</code>](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#multiplenegativesrankingloss) with these parameters:
+  | sentence_0                                                                                         | sentence_1                                                                                                                                                   | sentence_2                                                                                                                                                   |
+  |:---------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------|
+  | <code>How could changes in the emerging markets index (IEMG) affect my investment portfolio</code> | <code>[{"get_portfolio(None,True,None)": "portfolio"}, {"stress_test('portfolio','iemg',None,None)": "stress_test"}]</code>                                  | <code>[{"get_portfolio(None,True,None)": "portfolio"}, {"stress_test('portfolio','iemg',None,'up')": "stress_test"}]</code>                                  |
+  | <code>What role has the volatility factor played in my overall returns</code>                      | <code>[{"get_portfolio(None,True,None)": "portfolio"}, {"factor_contribution('portfolio','<DATES>','factor','volatility','returns')": "portfolio"}]</code>   | <code>[{"get_portfolio(None,True,None)": "portfolio"}, {"factor_contribution('portfolio','<DATES>','factor','volatility','portfolio')": "portfolio"}]</code> |
+  | <code>Is my portfolio overexposed to [A_REGION] country exposure</code>                            | <code>[{"get_portfolio(None,True,None)": "portfolio"}, {"factor_contribution('portfolio','<DATES>','region','<A_REGION>','portfolio')": "portfolio"}]</code> | <code>[{"get_portfolio(None,True,None)": "portfolio"}, {"factor_contribution('portfolio','<DATES>','theme','<A_THEME>','portfolio')": "portfolio"}]</code>   |
+* Loss: [<code>TripletLoss</code>](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#tripletloss) with these parameters:
   ```json
   {
-      "scale": 20.0,
-      "similarity_fct": "cos_sim"
+      "distance_metric": "TripletDistanceMetric.EUCLIDEAN",
+      "triplet_margin": 0.3
   }
   ```
 
@@ -319,7 +310,7 @@ You can finetune this model on your own dataset.
 ### Training Logs
 | Epoch  | Step | Training Loss |
 |:------:|:----:|:-------------:|
-| 8.9286 | 500  | 0.2737        |
+| 8.7719 | 500  | 0.0657        |
 
 
 ### Framework Versions
@@ -348,15 +339,15 @@ You can finetune this model on your own dataset.
 }
 ```
 
-#### MultipleNegativesRankingLoss
+#### TripletLoss
 ```bibtex
-@misc{henderson2017efficient,
-    title={Efficient Natural Language Response Suggestion for Smart Reply},
-    author={Matthew Henderson and Rami Al-Rfou and Brian Strope and Yun-hsuan Sung and Laszlo Lukacs and Ruiqi Guo and Sanjiv Kumar and Balint Miklos and Ray Kurzweil},
+@misc{hermans2017defense,
+    title={In Defense of the Triplet Loss for Person Re-Identification},
+    author={Alexander Hermans and Lucas Beyer and Bastian Leibe},
     year={2017},
-    eprint={1705.00652},
+    eprint={1703.07737},
     archivePrefix={arXiv},
-    primaryClass={cs.CL}
+    primaryClass={cs.CV}
 }
 ```
 

config.json

@@ -1,23 +1,30 @@
 {
   "architectures": [
-    "MPNetModel"
+    "BertModel"
   ],
   "attention_probs_dropout_prob": 0.1,
-  "bos_token_id": 0,
-  "eos_token_id": 2,
+  "classifier_dropout": null,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
-  "hidden_size": 768,
+  "hidden_size": 384,
+  "id2label": {
+    "0": "LABEL_0"
+  },
   "initializer_range": 0.02,
-  "intermediate_size": 3072,
-  "layer_norm_eps": 1e-05,
-  "max_position_embeddings": 514,
-  "model_type": "mpnet",
+  "intermediate_size": 1536,
+  "label2id": {
+    "LABEL_0": 0
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
   "num_attention_heads": 12,
   "num_hidden_layers": 12,
-  "pad_token_id": 1,
-  "relative_attention_num_buckets": 32,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
   "torch_dtype": "float32",
   "transformers_version": "4.50.0",
-  "vocab_size": 30527
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a0496292681a50dee1a691259069039679edf81e7b849fc1a242c23c8c46599d
-size 437967672
+oid sha256:18ac7948b64665eb9e4345c35b7735ca6456c30854498ad54f5bc6fc387b7b1c
+size 133462128

sentence_bert_config.json

@@ -1,4 +1,4 @@
 {
-  "max_seq_length": 384,
-  "do_lower_case": false
+  "max_seq_length": 512,
+  "do_lower_case": true
 }

special_tokens_map.json

@@ -1,41 +1,27 @@
 {
-  "bos_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
   "cls_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "</s>",
+    "content": "[CLS]",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "mask_token": {
-    "content": "<mask>",
-    "lstrip": true,
+    "content": "[MASK]",
+    "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "pad_token": {
-    "content": "<pad>",
+    "content": "[PAD]",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "sep_token": {
-    "content": "</s>",
+    "content": "[SEP]",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer_config.json

@@ -1,73 +1,58 @@
 {
   "added_tokens_decoder": {
     "0": {
-      "content": "<s>",
+      "content": "[PAD]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "1": {
-      "content": "<pad>",
+    "100": {
+      "content": "[UNK]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "2": {
-      "content": "</s>",
+    "101": {
+      "content": "[CLS]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "3": {
-      "content": "<unk>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "104": {
-      "content": "[UNK]",
+    "102": {
+      "content": "[SEP]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "30526": {
-      "content": "<mask>",
-      "lstrip": true,
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     }
   },
-  "bos_token": "<s>",
-  "clean_up_tokenization_spaces": false,
-  "cls_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
   "do_lower_case": true,
-  "eos_token": "</s>",
   "extra_special_tokens": {},
-  "mask_token": "<mask>",
-  "max_length": 128,
-  "model_max_length": 384,
-  "pad_to_multiple_of": null,
-  "pad_token": "<pad>",
-  "pad_token_type_id": 0,
-  "padding_side": "right",
-  "sep_token": "</s>",
-  "stride": 0,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
   "strip_accents": null,
   "tokenize_chinese_chars": true,
-  "tokenizer_class": "MPNetTokenizer",
-  "truncation_side": "right",
-  "truncation_strategy": "longest_first",
+  "tokenizer_class": "BertTokenizer",
   "unk_token": "[UNK]"
 }

vocab.txt

@@ -1,7 +1,3 @@
-<s>
-<pad>
-</s>
-<unk>
 [PAD]
 [unused0]
 [unused1]
@@ -30524,4 +30520,3 @@ necessitated
 ##：
 ##？
 ##～
-<mask>