Upload 23 files

Pretrained model on latex, Presentation MathML and Content MathML of formulas. It was introduced in https://link.springer.com/chapter/10.1007/978-981-99-7254-8_8

embedding_latex = model.encode([{'latex': latex}])
embedding_pmml = model.encode([{'mathml': pmml}])
embedding_cmml = model.encode([{'mathml': cmml}])


@inproceedings{wang2023math,
title={Math Information Retrieval with Contrastive Learning of Formula Embeddings},
author={Wang, Jingyi and Tian, Xuedong},
booktitle={International Conference on Web Information Systems Engineering},
pages={97--107},
year={2023},
organization={Springer}
}

0_Asym/140000857040976_MarkuplmTransformerForConMATH/added_tokens.json

@@ -0,0 +1,4 @@
+{
+  "<end-of-node>": 50266,
+  "[empty-title]": 50265
+}

0_Asym/140000857040976_MarkuplmTransformerForConMATH/config.json

@@ -0,0 +1,34 @@
+{
+  "_name_or_path": "output/training_markuplm_simcsemicrosoft-markuplm-base-2023-03-16_18-40-10",
+  "architectures": [
+    "MarkupLMModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_depth": 50,
+  "max_position_embeddings": 514,
+  "max_xpath_subs_unit_embeddings": 1024,
+  "max_xpath_tag_unit_embeddings": 256,
+  "model_type": "markuplm",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "subs_pad_id": 1001,
+  "tag_pad_id": 216,
+  "torch_dtype": "float32",
+  "transformers_version": "4.26.1",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 50267,
+  "xpath_unit_hidden_size": 32
+}

0_Asym/140000857040976_MarkuplmTransformerForConMATH/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "pad_token_id": 1,
+  "transformers_version": "4.26.1"
+}

0_Asym/140000857040976_MarkuplmTransformerForConMATH/preprocessor_config.json

@@ -0,0 +1,4 @@
+{
+  "feature_extractor_type": "MarkupLMFeatureExtractor",
+  "processor_class": "MarkupLMProcessor"
+}

0_Asym/140000857040976_MarkuplmTransformerForConMATH/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:644beecc9dcbf64a22d4a6058e21e564e47333f7384d16d7f85a1c43be7a4b0e
+size 540919261

0_Asym/140000857040976_MarkuplmTransformerForConMATH/sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 512,
+  "do_lower_case": false
+}

0_Asym/140000857040976_MarkuplmTransformerForConMATH/special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

0_Asym/140000857040976_MarkuplmTransformerForConMATH/tokenizer_config.json

@@ -0,0 +1,268 @@
+{
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "mask_token": {
+    "__type": "AddedToken",
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "max_depth": 50,
+  "max_width": 1000,
+  "model_max_length": 512,
+  "name_or_path": "output/training_markuplm_simcsemicrosoft-markuplm-base-2023-03-16_18-40-10",
+  "only_label_first_subword": true,
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token_label": -100,
+  "pad_width": 1001,
+  "processor_class": "MarkupLMProcessor",
+  "sep_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "special_tokens_map_file": null,
+  "tags_dict": {
+    "abs": 0,
+    "and": 1,
+    "annotation": 2,
+    "annotation-xml": 3,
+    "apply": 4,
+    "approx": 5,
+    "arccos": 6,
+    "arccosh": 7,
+    "arccot": 8,
+    "arccoth": 9,
+    "arccsc": 10,
+    "arccsch": 11,
+    "arcsec": 12,
+    "arcsech": 13,
+    "arcsin": 14,
+    "arcsinh": 15,
+    "arctan": 16,
+    "arctanh": 17,
+    "arg": 18,
+    "bind": 19,
+    "bvar": 20,
+    "card": 21,
+    "cartesianproduct": 22,
+    "cbytes": 23,
+    "ceiling": 24,
+    "cerror": 25,
+    "ci": 26,
+    "cn": 27,
+    "codomain": 28,
+    "complexes": 29,
+    "compose": 30,
+    "condition": 31,
+    "conjugate": 32,
+    "cos": 33,
+    "cosh": 34,
+    "cot": 35,
+    "coth": 36,
+    "cs": 37,
+    "csc": 38,
+    "csch": 39,
+    "csymbol": 40,
+    "curl": 41,
+    "declare": 42,
+    "degree": 43,
+    "determinant": 44,
+    "diff": 45,
+    "divergence": 46,
+    "divide": 47,
+    "domain": 48,
+    "domainofapplication": 49,
+    "emptyset": 50,
+    "eq": 51,
+    "equivalent": 52,
+    "eulergamma": 53,
+    "exists": 54,
+    "exp": 55,
+    "exponentiale": 56,
+    "factorial": 57,
+    "factorof": 58,
+    "false": 59,
+    "floor": 60,
+    "fn": 61,
+    "forall": 62,
+    "gcd": 63,
+    "geq": 64,
+    "grad": 65,
+    "gt": 66,
+    "ident": 67,
+    "image": 68,
+    "imaginary": 69,
+    "imaginaryi": 70,
+    "implies": 71,
+    "in": 72,
+    "infinity": 73,
+    "int": 74,
+    "integers": 75,
+    "intersect": 76,
+    "interval": 77,
+    "inverse": 78,
+    "lambda": 79,
+    "laplacian": 80,
+    "lcm": 81,
+    "leq": 82,
+    "limit": 83,
+    "list": 84,
+    "ln": 85,
+    "log": 86,
+    "logbase": 87,
+    "lowlimit": 88,
+    "lt": 89,
+    "maction": 90,
+    "maligngroup": 91,
+    "malignmark": 92,
+    "math": 93,
+    "matrix": 94,
+    "matrixrow": 95,
+    "max": 96,
+    "mean": 97,
+    "median": 98,
+    "menclose": 99,
+    "merror": 100,
+    "mfenced": 101,
+    "mfrac": 102,
+    "mglyph": 103,
+    "mi": 104,
+    "mi\"": 105,
+    "min": 106,
+    "minus": 107,
+    "mlabeledtr": 108,
+    "mlongdiv": 109,
+    "mmultiscripts": 110,
+    "mn": 111,
+    "mo": 112,
+    "mode": 113,
+    "moment": 114,
+    "momentabout": 115,
+    "mover": 116,
+    "mpadded": 117,
+    "mphantom": 118,
+    "mprescripts": 119,
+    "mroot": 120,
+    "mrow": 121,
+    "ms": 122,
+    "mscarries": 123,
+    "mscarry": 124,
+    "msgroup": 125,
+    "msline": 126,
+    "mspace": 127,
+    "msqrt": 128,
+    "msrow": 129,
+    "mstack": 130,
+    "mstyle": 131,
+    "msub": 132,
+    "msubsup": 133,
+    "msup": 134,
+    "mtable": 135,
+    "mtd": 136,
+    "mtext": 137,
+    "mtr": 138,
+    "munder": 139,
+    "munderover": 140,
+    "naturalnumbers": 141,
+    "neq": 142,
+    "none": 143,
+    "not": 144,
+    "notanumber": 145,
+    "notin": 146,
+    "notprsubset": 147,
+    "notsubset": 148,
+    "or": 149,
+    "otherwise": 150,
+    "outerproduct": 151,
+    "partialdiff": 152,
+    "pi": 153,
+    "piece": 154,
+    "piecewise": 155,
+    "plus": 156,
+    "power": 157,
+    "primes": 158,
+    "product": 159,
+    "prsubset": 160,
+    "quotient": 161,
+    "rationals": 162,
+    "real": 163,
+    "reals": 164,
+    "reln": 165,
+    "rem": 166,
+    "root": 167,
+    "scalarproduct": 168,
+    "sdev": 169,
+    "sec": 170,
+    "sech": 171,
+    "selector": 172,
+    "semantics": 173,
+    "sep": 174,
+    "set": 175,
+    "setdiff": 176,
+    "share": 177,
+    "sin": 178,
+    "sinh": 179,
+    "span": 180,
+    "subset": 181,
+    "sum": 182,
+    "tan": 183,
+    "tanh": 184,
+    "tendsto": 185,
+    "times": 186,
+    "transpose": 187,
+    "true": 188,
+    "union": 189,
+    "uplimit": 190,
+    "variance": 191,
+    "vector": 192,
+    "vectorproduct": 193,
+    "xor": 194
+  },
+  "tokenizer_class": "MarkupLMTokenizer",
+  "trim_offsets": false,
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

0_Asym/140000860578048_Transformer/config.json

@@ -0,0 +1,24 @@
+{
+  "_name_or_path": "sentence-transformers/all-mpnet-base-v2",
+  "architectures": [
+    "MPNetModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "mpnet",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "relative_attention_num_buckets": 32,
+  "torch_dtype": "float32",
+  "transformers_version": "4.26.1",
+  "vocab_size": 30527
+}

0_Asym/140000860578048_Transformer/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15d7a9813b0a24c11d46812f8bda297891490b64c6e7f47c228bc8dee06079f2
+size 438014769

0_Asym/140000860578048_Transformer/sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 512,
+  "do_lower_case": false
+}

0_Asym/140000860578048_Transformer/special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "[UNK]"
+}

0_Asym/140000860578048_Transformer/tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "do_lower_case": true,
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "model_max_length": 512,
+  "name_or_path": "sentence-transformers/all-mpnet-base-v2",
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "special_tokens_map_file": null,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "MPNetTokenizer",
+  "unk_token": "[UNK]"
+}

0_Asym/config.json

@@ -0,0 +1,17 @@
+{
+  "types": {
+    "140000860578048_Transformer": "sentence_transformers.models.Transformer",
+    "140000857040976_MarkuplmTransformerForConMATH": "sentence_transformers.models.MarkuplmTransformerForConMATH"
+  },
+  "structure": {
+    "latex": [
+      "140000860578048_Transformer"
+    ],
+    "mathml": [
+      "140000857040976_MarkuplmTransformerForConMATH"
+    ]
+  },
+  "parameters": {
+    "allow_empty_key": true
+  }
+}

1_Pooling/config.json

@@ -0,0 +1,9 @@
+{
+  "word_embedding_dimension": 768,
+  "pooling_mode_cls_token": false,
+  "pooling_mode_mean_tokens": true,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false,
+  "pooling_mode_weightedmean_tokens": false,
+  "pooling_mode_lasttoken": false
+}

README.md

@@ -1,3 +1,93 @@
 ---
-license: unknown
+pipeline_tag: sentence-similarity
+tags:
+- sentence-transformers
+- feature-extraction
+- sentence-similarity
+
 ---
+
+# {MODEL_NAME}
+
+This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 768 dimensional dense vector space and can be used for tasks like clustering or semantic search.
+
+<!--- Describe your model here -->
+
+## Usage (Sentence-Transformers)
+
+Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed:
+
+```
+pip install -U sentence-transformers
+```
+
+Then you can use the model like this:
+
+```python
+from sentence_transformers import SentenceTransformer
+sentences = ["This is an example sentence", "Each sentence is converted"]
+
+model = SentenceTransformer('{MODEL_NAME}')
+embeddings = model.encode(sentences)
+print(embeddings)
+```
+
+
+
+## Evaluation Results
+
+<!--- Describe how your model was evaluated -->
+
+For an automated evaluation of this model, see the *Sentence Embeddings Benchmark*: [https://seb.sbert.net](https://seb.sbert.net?model_name={MODEL_NAME})
+
+
+## Training
+The model was trained with the parameters:
+
+**DataLoader**:
+
+`torch.utils.data.dataloader.DataLoader` of length 45861 with parameters:
+```
+{'batch_size': 4, 'sampler': 'torch.utils.data.sampler.RandomSampler', 'batch_sampler': 'torch.utils.data.sampler.BatchSampler'}
+```
+
+**Loss**:
+
+`sentence_transformers.losses.MultipleNegativesSymmetricRankingLoss.MultipleNegativesSymmetricRankingLoss` with parameters:
+  ```
+  {'scale': 20.0, 'similarity_fct': 'cos_sim'}
+  ```
+
+Parameters of the fit()-Method:
+```
+{
+    "epochs": 1,
+    "evaluation_steps": 4586,
+    "evaluator": "sentence_transformers.evaluation.AlignmentandUniformityEvaluator.AlignmentandUniformityEvaluator",
+    "max_grad_norm": 1,
+    "optimizer_class": "<class 'torch.optim.adamw.AdamW'>",
+    "optimizer_params": {
+        "lr": 5e-05
+    },
+    "scheduler": "WarmupLinear",
+    "steps_per_epoch": null,
+    "warmup_steps": 4587,
+    "weight_decay": 0.01
+}
+```
+
+
+## Full Model Architecture
+```
+SentenceTransformer(
+  (0): Asym(
+    (latex-0): Transformer({'max_seq_length': 512, 'do_lower_case': False}) with Transformer model: MPNetModel 
+    (mathml-0): MarkuplmTransformer({'max_seq_length': 512, 'do_lower_case': False}) with Transformer model: MarkupLMModel 
+  )
+  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False})
+)
+```
+
+## Citing & Authors
+
+<!--- Describe where people can find more information -->

config_sentence_transformers.json

@@ -0,0 +1,7 @@
+{
+  "__version__": {
+    "sentence_transformers": "2.2.2",
+    "transformers": "4.26.1",
+    "pytorch": "1.12.1+cu113"
+  }
+}

modules.json

@@ -0,0 +1,14 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "0_Asym",
+    "type": "sentence_transformers.models.Asym"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  }
+]