End of training

README.md

@@ -0,0 +1,59 @@
+---
+base_model: LGAI-EXAONE/EXAONE-3.5-2.4B-Instruct
+datasets: testing_layer_29_Matrix_mlp.c_proj
+library_name: transformers
+model_name: output_exaone_tucker_rank16_newdata_b1_g2_gradient1_e1_b512
+tags:
+- generated_from_trainer
+- trl
+- sft
+licence: license
+---
+
+# Model Card for output_exaone_tucker_rank16_newdata_b1_g2_gradient1_e1_b512
+
+This model is a fine-tuned version of [LGAI-EXAONE/EXAONE-3.5-2.4B-Instruct](https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-2.4B-Instruct) on the [testing_layer_29_Matrix_mlp.c_proj](https://huggingface.co/datasets/testing_layer_29_Matrix_mlp.c_proj) dataset.
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="nileshmalpeddi/output_exaone_tucker_rank16_newdata_b1_g2_gradient1_e1_b512", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+ 
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- TRL: 0.15.2
+- Transformers: 4.46.3
+- Pytorch: 2.4.0
+- Datasets: 2.21.0
+- Tokenizers: 0.20.3
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallouédec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

all_results.json

@@ -0,0 +1,15 @@
+{
+    "eval_accuracy": 0.8084207240251016,
+    "eval_loss": 1.6845703125,
+    "eval_runtime": 933.2948,
+    "eval_samples": 25971,
+    "eval_samples_per_second": 27.827,
+    "eval_steps_per_second": 13.914,
+    "perplexity": 5.390134361325962,
+    "total_flos": 9.396165357758054e+17,
+    "train_loss": 1.722812063241433,
+    "train_runtime": 12082.0087,
+    "train_samples": 294393,
+    "train_samples_per_second": 24.366,
+    "train_steps_per_second": 12.183
+}

config.json

@@ -0,0 +1,39 @@
+{
+  "_name_or_path": "LGAI-EXAONE/EXAONE-3.5-2.4B-Instruct",
+  "activation_function": "silu",
+  "architectures": [
+    "ExaoneForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "LGAI-EXAONE/EXAONE-3.5-2.4B-Instruct--configuration_exaone.ExaoneConfig",
+    "AutoModelForCausalLM": "LGAI-EXAONE/EXAONE-3.5-2.4B-Instruct--modeling_exaone.ExaoneForCausalLM",
+    "AutoModelForSequenceClassification": "LGAI-EXAONE/EXAONE-3.5-2.4B-Instruct--modeling_exaone.ExaoneForSequenceClassification"
+  },
+  "bos_token_id": 1,
+  "embed_dropout": 0.0,
+  "eos_token_id": 361,
+  "head_dim": 80,
+  "hidden_size": 2560,
+  "initializer_range": 0.02,
+  "intermediate_size": 7168,
+  "layer_norm_epsilon": 1e-05,
+  "max_position_embeddings": 32768,
+  "model_type": "exaone",
+  "num_attention_heads": 32,
+  "num_key_value_heads": 8,
+  "num_layers": 30,
+  "pad_token_id": 0,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 1000000,
+  "torch_dtype": "float16",
+  "transformers_version": "4.46.3",
+  "use_cache": true,
+  "vocab_size": 102400
+}

eval_layer_index_29_mat_mlp.c_proj_results.json

@@ -0,0 +1,9 @@
+{
+    "eval_accuracy": 0.8084207240251016,
+    "eval_loss": 1.6845703125,
+    "eval_runtime": 933.2948,
+    "eval_samples": 25971,
+    "eval_samples_per_second": 27.827,
+    "eval_steps_per_second": 13.914,
+    "perplexity": 5.390134361325962
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 361,
+  "pad_token_id": 0,
+  "transformers_version": "4.46.3"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d7f8ac55165156d24fd68aee4367a9732cf51e950a3f1e760a2a14a26fae8da
+size 4701521288

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a21b63c2c74dfd9fb217aa27218979d0e67ba9d7a8703dedde6c6f7f64dc4bc8
+size 524288128

model.safetensors.index.json

@@ -0,0 +1,286 @@
+{
+  "metadata": {
+    "total_size": 5225777664
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.0.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.14.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.14.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.14.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.14.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.14.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.14.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.14.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.14.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.14.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.15.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.15.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.15.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.15.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.15.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.15.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.15.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.15.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.15.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.16.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.16.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.16.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.16.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.16.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.16.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.16.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.16.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.16.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.17.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.17.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.17.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.17.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.17.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.17.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.17.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.17.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.17.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.18.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.18.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.18.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.18.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.18.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.18.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.18.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.18.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.18.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.19.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.19.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.19.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.19.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.19.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.19.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.19.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.19.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.19.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.20.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.20.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.20.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.20.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.20.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.20.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.20.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.20.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.20.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.21.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.21.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.21.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.21.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.21.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.21.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.21.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.21.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.21.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.22.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.22.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.22.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.22.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.22.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.22.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.22.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.22.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.22.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.23.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.23.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.23.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.23.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.23.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.23.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.23.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.23.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.23.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.24.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.24.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.24.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.24.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.24.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.24.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.24.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.24.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.24.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.25.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.25.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.25.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.25.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.25.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.25.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.25.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.25.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.25.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.26.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.26.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.26.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.26.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.26.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.26.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.26.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.26.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.26.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.27.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.27.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.27.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.27.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.27.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.27.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.27.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.27.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.27.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.28.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.28.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.28.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.28.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.28.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.28.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.28.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.28.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.28.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.29.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.29.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.29.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.29.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.29.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.29.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.29.mlp.c_fc_0.tucker_core": "model-00001-of-00002.safetensors",
+    "transformer.h.29.mlp.c_fc_0.tucker_factors.0": "model-00001-of-00002.safetensors",
+    "transformer.h.29.mlp.c_fc_0.tucker_factors.1": "model-00001-of-00002.safetensors",
+    "transformer.h.29.mlp.c_fc_1.tucker_core": "model-00001-of-00002.safetensors",
+    "transformer.h.29.mlp.c_fc_1.tucker_factors.0": "model-00001-of-00002.safetensors",
+    "transformer.h.29.mlp.c_fc_1.tucker_factors.1": "model-00001-of-00002.safetensors",
+    "transformer.h.29.mlp.c_proj.tucker_core": "model-00001-of-00002.safetensors",
+    "transformer.h.29.mlp.c_proj.tucker_factors.0": "model-00001-of-00002.safetensors",
+    "transformer.h.29.mlp.c_proj.tucker_factors.1": "model-00001-of-00002.safetensors",
+    "transformer.h.3.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.ln_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.ln_2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.ln_f.weight": "model-00001-of-00002.safetensors",
+    "transformer.wte.weight": "model-00001-of-00002.safetensors"
+  }
+}

runs/Mar15_01-18-11_demo2/events.out.tfevents.1742027606.demo2.212607.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39b05f0c48c75fa18441426cc73edbfefb5167a6084e879879ae930a8ae48596
+size 88628

runs/Mar15_01-18-11_demo2/events.out.tfevents.1742040651.demo2.212607.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81ebc54830d80e646b422d40273956c0ff1c7d4a1e77bbe746424269ac0f0e38
+size 481

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "[BOS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "[|endofturn|]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,3221 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[BOS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[EOS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "                               ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "5": {
+      "content": "                              ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "6": {
+      "content": "                             ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "7": {
+      "content": "                            ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "8": {
+      "content": "                           ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "9": {
+      "content": "                          ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "10": {
+      "content": "                         ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "11": {
+      "content": "                        ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "12": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "13": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "14": {
+      "content": "                     ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "15": {
+      "content": "                    ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "16": {
+      "content": "                   ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "17": {
+      "content": "                  ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "18": {
+      "content": "                 ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "19": {
+      "content": "                ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "20": {
+      "content": "               ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "21": {
+      "content": "              ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "22": {
+      "content": "             ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "23": {
+      "content": "            ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "24": {
+      "content": "           ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "25": {
+      "content": "          ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "26": {
+      "content": "         ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "27": {
+      "content": "        ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "28": {
+      "content": "       ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "29": {
+      "content": "      ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "30": {
+      "content": "     ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "31": {
+      "content": "    ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "32": {
+      "content": "   ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "33": {
+      "content": "  ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "34": {
+      "content": "\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "35": {
+      "content": "\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "36": {
+      "content": "\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "37": {
+      "content": "\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "38": {
+      "content": "\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "39": {
+      "content": "\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "40": {
+      "content": "\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "41": {
+      "content": "\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "42": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "43": {
+      "content": "<|c|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "44": {
+      "content": "<|c++|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "45": {
+      "content": "<|python|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "46": {
+      "content": "<|javascript|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "47": {
+      "content": "<|markdown|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48": {
+      "content": "<|html|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49": {
+      "content": "<|css|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50": {
+      "content": "<|vue|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "51": {
+      "content": "<|java|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "52": {
+      "content": "PI:URL",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "53": {
+      "content": "PI:EMAIL",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "54": {
+      "content": "PI:ACCOUNT_NUM",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "55": {
+      "content": "PI:PHONE_NUM",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "56": {
+      "content": "PI:BUSINESS_NUM",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "57": {
+      "content": "PI:ANNON",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "58": {
+      "content": "PI:KEY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "59": {
+      "content": "PI:ID",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "60": {
+      "content": "PI:IP_ADDRESS",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "61": {
+      "content": "PI:USER",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "62": {
+      "content": "[unused0]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "63": {
+      "content": "[unused1]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64": {
+      "content": "[unused2]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65": {
+      "content": "[unused3]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "66": {
+      "content": "[unused4]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "67": {
+      "content": "[unused5]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "68": {
+      "content": "[unused6]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "69": {
+      "content": "[unused7]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "70": {
+      "content": "[unused8]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "71": {
+      "content": "[unused9]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "72": {
+      "content": "[unused10]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "73": {
+      "content": "[unused11]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "74": {
+      "content": "[unused12]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "75": {
+      "content": "[unused13]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "76": {
+      "content": "[unused14]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "77": {
+      "content": "[unused15]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "78": {
+      "content": "[unused16]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "79": {
+      "content": "[unused17]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "80": {
+      "content": "[unused18]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "81": {
+      "content": "[unused19]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "82": {
+      "content": "[unused20]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "83": {
+      "content": "[unused21]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "84": {
+      "content": "[unused22]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "85": {
+      "content": "[unused23]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "86": {
+      "content": "[unused24]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "87": {
+      "content": "[unused25]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "88": {
+      "content": "[unused26]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "89": {
+      "content": "[unused27]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "90": {
+      "content": "[unused28]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "91": {
+      "content": "[unused29]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92": {
+      "content": "[unused30]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "93": {
+      "content": "[unused31]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "94": {
+      "content": "[unused32]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "95": {
+      "content": "[unused33]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "96": {
+      "content": "[unused34]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "97": {
+      "content": "[unused35]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "98": {
+      "content": "[unused36]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "99": {
+      "content": "[unused37]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[unused38]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[unused39]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[unused40]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[unused41]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "104": {
+      "content": "[unused42]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "105": {
+      "content": "[unused43]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "106": {
+      "content": "[unused44]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "[unused45]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "108": {
+      "content": "[unused46]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "109": {
+      "content": "[unused47]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110": {
+      "content": "[unused48]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "111": {
+      "content": "[unused49]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "112": {
+      "content": "[unused50]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "113": {
+      "content": "[unused51]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "114": {
+      "content": "[unused52]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "115": {
+      "content": "[unused53]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "116": {
+      "content": "[unused54]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "117": {
+      "content": "[unused55]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "118": {
+      "content": "[unused56]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "119": {
+      "content": "[unused57]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "120": {
+      "content": "[unused58]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "121": {
+      "content": "[unused59]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "122": {
+      "content": "[unused60]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "123": {
+      "content": "[unused61]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "124": {
+      "content": "[unused62]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "125": {
+      "content": "[unused63]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "126": {
+      "content": "[unused64]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "127": {
+      "content": "[unused65]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128": {
+      "content": "[unused66]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "129": {
+      "content": "[unused67]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "130": {
+      "content": "[unused68]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "131": {
+      "content": "[unused69]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "132": {
+      "content": "[unused70]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "133": {
+      "content": "[unused71]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "134": {
+      "content": "[unused72]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "135": {
+      "content": "[unused73]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "136": {
+      "content": "[unused74]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "137": {
+      "content": "[unused75]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "138": {
+      "content": "[unused76]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "139": {
+      "content": "[unused77]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "140": {
+      "content": "[unused78]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "141": {
+      "content": "[unused79]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "142": {
+      "content": "[unused80]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "143": {
+      "content": "[unused81]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "144": {
+      "content": "[unused82]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "145": {
+      "content": "[unused83]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "146": {
+      "content": "[unused84]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "147": {
+      "content": "[unused85]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "148": {
+      "content": "[unused86]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "149": {
+      "content": "[unused87]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "150": {
+      "content": "[unused88]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151": {
+      "content": "[unused89]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152": {
+      "content": "[unused90]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "153": {
+      "content": "[unused91]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "154": {
+      "content": "[unused92]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "155": {
+      "content": "[unused93]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156": {
+      "content": "[unused94]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157": {
+      "content": "[unused95]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "158": {
+      "content": "[unused96]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "159": {
+      "content": "[unused97]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "160": {
+      "content": "[unused98]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "161": {
+      "content": "[unused99]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "162": {
+      "content": "[extra_id_0]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "163": {
+      "content": "[extra_id_1]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "164": {
+      "content": "[extra_id_2]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "165": {
+      "content": "[extra_id_3]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "166": {
+      "content": "[extra_id_4]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "167": {
+      "content": "[extra_id_5]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "168": {
+      "content": "[extra_id_6]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "169": {
+      "content": "[extra_id_7]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "170": {
+      "content": "[extra_id_8]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "171": {
+      "content": "[extra_id_9]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "172": {
+      "content": "[extra_id_10]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "173": {
+      "content": "[extra_id_11]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "174": {
+      "content": "[extra_id_12]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "175": {
+      "content": "[extra_id_13]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "176": {
+      "content": "[extra_id_14]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "177": {
+      "content": "[extra_id_15]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "178": {
+      "content": "[extra_id_16]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "179": {
+      "content": "[extra_id_17]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "180": {
+      "content": "[extra_id_18]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "181": {
+      "content": "[extra_id_19]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "182": {
+      "content": "[extra_id_20]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "183": {
+      "content": "[extra_id_21]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "184": {
+      "content": "[extra_id_22]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "185": {
+      "content": "[extra_id_23]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "186": {
+      "content": "[extra_id_24]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "187": {
+      "content": "[extra_id_25]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "188": {
+      "content": "[extra_id_26]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "189": {
+      "content": "[extra_id_27]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "190": {
+      "content": "[extra_id_28]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "191": {
+      "content": "[extra_id_29]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "192": {
+      "content": "[extra_id_30]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "193": {
+      "content": "[extra_id_31]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "194": {
+      "content": "[extra_id_32]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "195": {
+      "content": "[extra_id_33]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "196": {
+      "content": "[extra_id_34]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "197": {
+      "content": "[extra_id_35]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "198": {
+      "content": "[extra_id_36]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "199": {
+      "content": "[extra_id_37]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "200": {
+      "content": "[extra_id_38]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "201": {
+      "content": "[extra_id_39]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "202": {
+      "content": "[extra_id_40]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "203": {
+      "content": "[extra_id_41]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "204": {
+      "content": "[extra_id_42]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "205": {
+      "content": "[extra_id_43]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "206": {
+      "content": "[extra_id_44]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "207": {
+      "content": "[extra_id_45]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "208": {
+      "content": "[extra_id_46]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "209": {
+      "content": "[extra_id_47]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "210": {
+      "content": "[extra_id_48]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "211": {
+      "content": "[extra_id_49]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "212": {
+      "content": "[extra_id_50]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "213": {
+      "content": "[extra_id_51]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "214": {
+      "content": "[extra_id_52]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "215": {
+      "content": "[extra_id_53]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "216": {
+      "content": "[extra_id_54]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "217": {
+      "content": "[extra_id_55]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "218": {
+      "content": "[extra_id_56]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "219": {
+      "content": "[extra_id_57]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "220": {
+      "content": "[extra_id_58]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "221": {
+      "content": "[extra_id_59]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "222": {
+      "content": "[extra_id_60]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "223": {
+      "content": "[extra_id_61]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "224": {
+      "content": "[extra_id_62]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "225": {
+      "content": "[extra_id_63]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "226": {
+      "content": "[extra_id_64]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "227": {
+      "content": "[extra_id_65]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "228": {
+      "content": "[extra_id_66]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "229": {
+      "content": "[extra_id_67]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "230": {
+      "content": "[extra_id_68]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "231": {
+      "content": "[extra_id_69]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "232": {
+      "content": "[extra_id_70]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "233": {
+      "content": "[extra_id_71]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "234": {
+      "content": "[extra_id_72]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "235": {
+      "content": "[extra_id_73]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "236": {
+      "content": "[extra_id_74]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "237": {
+      "content": "[extra_id_75]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "238": {
+      "content": "[extra_id_76]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "239": {
+      "content": "[extra_id_77]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "240": {
+      "content": "[extra_id_78]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "241": {
+      "content": "[extra_id_79]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "242": {
+      "content": "[extra_id_80]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "243": {
+      "content": "[extra_id_81]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "244": {
+      "content": "[extra_id_82]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "245": {
+      "content": "[extra_id_83]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "246": {
+      "content": "[extra_id_84]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "247": {
+      "content": "[extra_id_85]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "248": {
+      "content": "[extra_id_86]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "249": {
+      "content": "[extra_id_87]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250": {
+      "content": "[extra_id_88]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "251": {
+      "content": "[extra_id_89]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "252": {
+      "content": "[extra_id_90]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "253": {
+      "content": "[extra_id_91]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "254": {
+      "content": "[extra_id_92]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "255": {
+      "content": "[extra_id_93]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "256": {
+      "content": "[extra_id_94]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "257": {
+      "content": "[extra_id_95]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "258": {
+      "content": "[extra_id_96]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "259": {
+      "content": "[extra_id_97]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "260": {
+      "content": "[extra_id_98]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "261": {
+      "content": "[extra_id_99]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "262": {
+      "content": "[extra_id_100]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "263": {
+      "content": "[extra_id_101]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "264": {
+      "content": "[extra_id_102]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "265": {
+      "content": "[extra_id_103]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "266": {
+      "content": "[extra_id_104]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "267": {
+      "content": "[extra_id_105]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "268": {
+      "content": "[extra_id_106]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "269": {
+      "content": "[extra_id_107]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "270": {
+      "content": "[extra_id_108]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "271": {
+      "content": "[extra_id_109]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "272": {
+      "content": "[extra_id_110]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "273": {
+      "content": "[extra_id_111]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "274": {
+      "content": "[extra_id_112]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "275": {
+      "content": "[extra_id_113]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "276": {
+      "content": "[extra_id_114]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "277": {
+      "content": "[extra_id_115]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "278": {
+      "content": "[extra_id_116]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "279": {
+      "content": "[extra_id_117]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "280": {
+      "content": "[extra_id_118]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "281": {
+      "content": "[extra_id_119]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "282": {
+      "content": "[extra_id_120]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "283": {
+      "content": "[extra_id_121]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "284": {
+      "content": "[extra_id_122]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "285": {
+      "content": "[extra_id_123]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "286": {
+      "content": "[extra_id_124]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "287": {
+      "content": "[extra_id_125]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "288": {
+      "content": "[extra_id_126]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "289": {
+      "content": "[extra_id_127]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "290": {
+      "content": "[extra_id_128]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "291": {
+      "content": "[extra_id_129]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "292": {
+      "content": "[extra_id_130]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "293": {
+      "content": "[extra_id_131]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "294": {
+      "content": "[extra_id_132]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "295": {
+      "content": "[extra_id_133]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "296": {
+      "content": "[extra_id_134]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "297": {
+      "content": "[extra_id_135]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "298": {
+      "content": "[extra_id_136]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "299": {
+      "content": "[extra_id_137]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "300": {
+      "content": "[extra_id_138]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "301": {
+      "content": "[extra_id_139]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "302": {
+      "content": "[extra_id_140]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "303": {
+      "content": "[extra_id_141]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "304": {
+      "content": "[extra_id_142]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "305": {
+      "content": "[extra_id_143]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "306": {
+      "content": "[extra_id_144]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "307": {
+      "content": "[extra_id_145]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "308": {
+      "content": "[extra_id_146]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "309": {
+      "content": "[extra_id_147]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "310": {
+      "content": "[extra_id_148]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "311": {
+      "content": "[extra_id_149]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "312": {
+      "content": "[extra_id_150]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "313": {
+      "content": "[extra_id_151]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "314": {
+      "content": "[extra_id_152]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "315": {
+      "content": "[extra_id_153]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "316": {
+      "content": "[extra_id_154]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "317": {
+      "content": "[extra_id_155]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "318": {
+      "content": "[extra_id_156]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "319": {
+      "content": "[extra_id_157]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "320": {
+      "content": "[extra_id_158]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "321": {
+      "content": "[extra_id_159]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "322": {
+      "content": "[extra_id_160]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "323": {
+      "content": "[extra_id_161]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "324": {
+      "content": "[extra_id_162]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "325": {
+      "content": "[extra_id_163]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "326": {
+      "content": "[extra_id_164]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "327": {
+      "content": "[extra_id_165]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "328": {
+      "content": "[extra_id_166]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "329": {
+      "content": "[extra_id_167]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "330": {
+      "content": "[extra_id_168]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "331": {
+      "content": "[extra_id_169]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "332": {
+      "content": "[extra_id_170]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "333": {
+      "content": "[extra_id_171]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "334": {
+      "content": "[extra_id_172]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "335": {
+      "content": "[extra_id_173]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "336": {
+      "content": "[extra_id_174]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "337": {
+      "content": "[extra_id_175]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "338": {
+      "content": "[extra_id_176]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "339": {
+      "content": "[extra_id_177]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "340": {
+      "content": "[extra_id_178]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "341": {
+      "content": "[extra_id_179]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "342": {
+      "content": "[extra_id_180]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "343": {
+      "content": "[extra_id_181]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "344": {
+      "content": "[extra_id_182]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "345": {
+      "content": "[extra_id_183]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "346": {
+      "content": "[extra_id_184]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "347": {
+      "content": "[extra_id_185]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "348": {
+      "content": "[extra_id_186]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "349": {
+      "content": "[extra_id_187]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "350": {
+      "content": "[extra_id_188]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "351": {
+      "content": "[extra_id_189]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "352": {
+      "content": "[extra_id_190]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "353": {
+      "content": "[extra_id_191]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "354": {
+      "content": "[extra_id_192]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "355": {
+      "content": "[extra_id_193]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "356": {
+      "content": "[extra_id_194]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "357": {
+      "content": "[extra_id_195]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "358": {
+      "content": "[extra_id_196]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "359": {
+      "content": "[extra_id_197]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "360": {
+      "content": "[extra_id_198]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "361": {
+      "content": "[|endofturn|]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_token": [
+    "[unused0]",
+    "[unused1]",
+    "[unused2]",
+    "[unused3]",
+    "[unused4]",
+    "[unused5]",
+    "[unused6]",
+    "[unused7]",
+    "[unused8]",
+    "[unused9]",
+    "[unused10]",
+    "[unused11]",
+    "[unused12]",
+    "[unused13]",
+    "[unused14]",
+    "[unused15]",
+    "[unused16]",
+    "[unused17]",
+    "[unused18]",
+    "[unused19]",
+    "[unused20]",
+    "[unused21]",
+    "[unused22]",
+    "[unused23]",
+    "[unused24]",
+    "[unused25]",
+    "[unused26]",
+    "[unused27]",
+    "[unused28]",
+    "[unused29]",
+    "[unused30]",
+    "[unused31]",
+    "[unused32]",
+    "[unused33]",
+    "[unused34]",
+    "[unused35]",
+    "[unused36]",
+    "[unused37]",
+    "[unused38]",
+    "[unused39]",
+    "[unused40]",
+    "[unused41]",
+    "[unused42]",
+    "[unused43]",
+    "[unused44]",
+    "[unused45]",
+    "[unused46]",
+    "[unused47]",
+    "[unused48]",
+    "[unused49]",
+    "[unused50]",
+    "[unused51]",
+    "[unused52]",
+    "[unused53]",
+    "[unused54]",
+    "[unused55]",
+    "[unused56]",
+    "[unused57]",
+    "[unused58]",
+    "[unused59]",
+    "[unused60]",
+    "[unused61]",
+    "[unused62]",
+    "[unused63]",
+    "[unused64]",
+    "[unused65]",
+    "[unused66]",
+    "[unused67]",
+    "[unused68]",
+    "[unused69]",
+    "[unused70]",
+    "[unused71]",
+    "[unused72]",
+    "[unused73]",
+    "[unused74]",
+    "[unused75]",
+    "[unused76]",
+    "[unused77]",
+    "[unused78]",
+    "[unused79]",
+    "[unused80]",
+    "[unused81]",
+    "[unused82]",
+    "[unused83]",
+    "[unused84]",
+    "[unused85]",
+    "[unused86]",
+    "[unused87]",
+    "[unused88]",
+    "[unused89]",
+    "[unused90]",
+    "[unused91]",
+    "[unused92]",
+    "[unused93]",
+    "[unused94]",
+    "[unused95]",
+    "[unused96]",
+    "[unused97]",
+    "[unused98]",
+    "[unused99]",
+    "[extra_id_0]",
+    "[extra_id_1]",
+    "[extra_id_2]",
+    "[extra_id_3]",
+    "[extra_id_4]",
+    "[extra_id_5]",
+    "[extra_id_6]",
+    "[extra_id_7]",
+    "[extra_id_8]",
+    "[extra_id_9]",
+    "[extra_id_10]",
+    "[extra_id_11]",
+    "[extra_id_12]",
+    "[extra_id_13]",
+    "[extra_id_14]",
+    "[extra_id_15]",
+    "[extra_id_16]",
+    "[extra_id_17]",
+    "[extra_id_18]",
+    "[extra_id_19]",
+    "[extra_id_20]",
+    "[extra_id_21]",
+    "[extra_id_22]",
+    "[extra_id_23]",
+    "[extra_id_24]",
+    "[extra_id_25]",
+    "[extra_id_26]",
+    "[extra_id_27]",
+    "[extra_id_28]",
+    "[extra_id_29]",
+    "[extra_id_30]",
+    "[extra_id_31]",
+    "[extra_id_32]",
+    "[extra_id_33]",
+    "[extra_id_34]",
+    "[extra_id_35]",
+    "[extra_id_36]",
+    "[extra_id_37]",
+    "[extra_id_38]",
+    "[extra_id_39]",
+    "[extra_id_40]",
+    "[extra_id_41]",
+    "[extra_id_42]",
+    "[extra_id_43]",
+    "[extra_id_44]",
+    "[extra_id_45]",
+    "[extra_id_46]",
+    "[extra_id_47]",
+    "[extra_id_48]",
+    "[extra_id_49]",
+    "[extra_id_50]",
+    "[extra_id_51]",
+    "[extra_id_52]",
+    "[extra_id_53]",
+    "[extra_id_54]",
+    "[extra_id_55]",
+    "[extra_id_56]",
+    "[extra_id_57]",
+    "[extra_id_58]",
+    "[extra_id_59]",
+    "[extra_id_60]",
+    "[extra_id_61]",
+    "[extra_id_62]",
+    "[extra_id_63]",
+    "[extra_id_64]",
+    "[extra_id_65]",
+    "[extra_id_66]",
+    "[extra_id_67]",
+    "[extra_id_68]",
+    "[extra_id_69]",
+    "[extra_id_70]",
+    "[extra_id_71]",
+    "[extra_id_72]",
+    "[extra_id_73]",
+    "[extra_id_74]",
+    "[extra_id_75]",
+    "[extra_id_76]",
+    "[extra_id_77]",
+    "[extra_id_78]",
+    "[extra_id_79]",
+    "[extra_id_80]",
+    "[extra_id_81]",
+    "[extra_id_82]",
+    "[extra_id_83]",
+    "[extra_id_84]",
+    "[extra_id_85]",
+    "[extra_id_86]",
+    "[extra_id_87]",
+    "[extra_id_88]",
+    "[extra_id_89]",
+    "[extra_id_90]",
+    "[extra_id_91]",
+    "[extra_id_92]",
+    "[extra_id_93]",
+    "[extra_id_94]",
+    "[extra_id_95]",
+    "[extra_id_96]",
+    "[extra_id_97]",
+    "[extra_id_98]",
+    "[extra_id_99]",
+    "[extra_id_100]",
+    "[extra_id_101]",
+    "[extra_id_102]",
+    "[extra_id_103]",
+    "[extra_id_104]",
+    "[extra_id_105]",
+    "[extra_id_106]",
+    "[extra_id_107]",
+    "[extra_id_108]",
+    "[extra_id_109]",
+    "[extra_id_110]",
+    "[extra_id_111]",
+    "[extra_id_112]",
+    "[extra_id_113]",
+    "[extra_id_114]",
+    "[extra_id_115]",
+    "[extra_id_116]",
+    "[extra_id_117]",
+    "[extra_id_118]",
+    "[extra_id_119]",
+    "[extra_id_120]",
+    "[extra_id_121]",
+    "[extra_id_122]",
+    "[extra_id_123]",
+    "[extra_id_124]",
+    "[extra_id_125]",
+    "[extra_id_126]",
+    "[extra_id_127]",
+    "[extra_id_128]",
+    "[extra_id_129]",
+    "[extra_id_130]",
+    "[extra_id_131]",
+    "[extra_id_132]",
+    "[extra_id_133]",
+    "[extra_id_134]",
+    "[extra_id_135]",
+    "[extra_id_136]",
+    "[extra_id_137]",
+    "[extra_id_138]",
+    "[extra_id_139]",
+    "[extra_id_140]",
+    "[extra_id_141]",
+    "[extra_id_142]",
+    "[extra_id_143]",
+    "[extra_id_144]",
+    "[extra_id_145]",
+    "[extra_id_146]",
+    "[extra_id_147]",
+    "[extra_id_148]",
+    "[extra_id_149]",
+    "[extra_id_150]",
+    "[extra_id_151]",
+    "[extra_id_152]",
+    "[extra_id_153]",
+    "[extra_id_154]",
+    "[extra_id_155]",
+    "[extra_id_156]",
+    "[extra_id_157]",
+    "[extra_id_158]",
+    "[extra_id_159]",
+    "[extra_id_160]",
+    "[extra_id_161]",
+    "[extra_id_162]",
+    "[extra_id_163]",
+    "[extra_id_164]",
+    "[extra_id_165]",
+    "[extra_id_166]",
+    "[extra_id_167]",
+    "[extra_id_168]",
+    "[extra_id_169]",
+    "[extra_id_170]",
+    "[extra_id_171]",
+    "[extra_id_172]",
+    "[extra_id_173]",
+    "[extra_id_174]",
+    "[extra_id_175]",
+    "[extra_id_176]",
+    "[extra_id_177]",
+    "[extra_id_178]",
+    "[extra_id_179]",
+    "[extra_id_180]",
+    "[extra_id_181]",
+    "[extra_id_182]",
+    "[extra_id_183]",
+    "[extra_id_184]",
+    "[extra_id_185]",
+    "[extra_id_186]",
+    "[extra_id_187]",
+    "[extra_id_188]",
+    "[extra_id_189]",
+    "[extra_id_190]",
+    "[extra_id_191]",
+    "[extra_id_192]",
+    "[extra_id_193]",
+    "[extra_id_194]",
+    "[extra_id_195]",
+    "[extra_id_196]",
+    "[extra_id_197]",
+    "[extra_id_198]",
+    "[|endofturn|]",
+    "PI:URL",
+    "PI:EMAIL",
+    "PI:ACCOUNT_NUM",
+    "PI:PHONE_NUM",
+    "PI:BUSINESS_NUM",
+    "PI:ANNON",
+    "PI:KEY",
+    "PI:ID",
+    "PI:IP_ADDRESS",
+    "PI:USER"
+  ],
+  "bos_token": "[BOS]",
+  "chat_template": "{% for message in messages %}{% if loop.first and message['role'] != 'system' %}{{ '[|system|][|endofturn|]\n' }}{% endif %}{{ '[|' + message['role'] + '|]' + message['content'] }}{% if message['role'] == 'user' %}{{ '\n' }}{% else %}{{ '[|endofturn|]\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '[|assistant|]' }}{% endif %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "[|endofturn|]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "[UNK]"
+}

train_layer_index_29_mat_mlp.c_proj_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 9.396165357758054e+17,
+    "train_loss": 1.722812063241433,
+    "train_runtime": 12082.0087,
+    "train_samples": 294393,
+    "train_samples_per_second": 24.366,
+    "train_steps_per_second": 12.183
+}

trainer_state.json

@@ -0,0 +1,2395 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 147197,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0033968083588660097,
+      "grad_norm": 3.8452000617980957,
+      "learning_rate": 0.0004983185798623613,
+      "loss": 2.0868,
+      "mean_token_accuracy": 0.5478256906867027,
+      "step": 500
+    },
+    {
+      "epoch": 0.006793616717732019,
+      "grad_norm": 2.1202304363250732,
+      "learning_rate": 0.0004966201756829283,
+      "loss": 1.9203,
+      "mean_token_accuracy": 0.5669565772414208,
+      "step": 1000
+    },
+    {
+      "epoch": 0.010190425076598028,
+      "grad_norm": 2.57440185546875,
+      "learning_rate": 0.0004949217715034954,
+      "loss": 1.8892,
+      "mean_token_accuracy": 0.5712102488875389,
+      "step": 1500
+    },
+    {
+      "epoch": 0.013587233435464039,
+      "grad_norm": 1.7826733589172363,
+      "learning_rate": 0.0004932233673240623,
+      "loss": 1.8581,
+      "mean_token_accuracy": 0.574765417098999,
+      "step": 2000
+    },
+    {
+      "epoch": 0.016984041794330048,
+      "grad_norm": 2.433124542236328,
+      "learning_rate": 0.0004915249631446293,
+      "loss": 1.8426,
+      "mean_token_accuracy": 0.5774432135820389,
+      "step": 2500
+    },
+    {
+      "epoch": 0.020380850153196056,
+      "grad_norm": 2.0346601009368896,
+      "learning_rate": 0.0004898265589651962,
+      "loss": 1.8353,
+      "mean_token_accuracy": 0.5788127770423889,
+      "step": 3000
+    },
+    {
+      "epoch": 0.023777658512062066,
+      "grad_norm": 1.5259729623794556,
+      "learning_rate": 0.0004881281547857633,
+      "loss": 1.8264,
+      "mean_token_accuracy": 0.5811644524335862,
+      "step": 3500
+    },
+    {
+      "epoch": 0.027174466870928077,
+      "grad_norm": 1.6687489748001099,
+      "learning_rate": 0.0004864297506063303,
+      "loss": 1.8275,
+      "mean_token_accuracy": 0.5799615281820297,
+      "step": 4000
+    },
+    {
+      "epoch": 0.030571275229794085,
+      "grad_norm": 1.838523268699646,
+      "learning_rate": 0.00048473814004361507,
+      "loss": 1.8073,
+      "mean_token_accuracy": 0.5813801457881927,
+      "step": 4500
+    },
+    {
+      "epoch": 0.033968083588660096,
+      "grad_norm": 1.710098147392273,
+      "learning_rate": 0.00048303973586418203,
+      "loss": 1.822,
+      "mean_token_accuracy": 0.5785296305418014,
+      "step": 5000
+    },
+    {
+      "epoch": 0.037364891947526103,
+      "grad_norm": 1.9218050241470337,
+      "learning_rate": 0.0004813481253014668,
+      "loss": 1.8267,
+      "mean_token_accuracy": 0.5784759641885757,
+      "step": 5500
+    },
+    {
+      "epoch": 0.04076170030639211,
+      "grad_norm": 2.2939748764038086,
+      "learning_rate": 0.00047964972112203374,
+      "loss": 1.8119,
+      "mean_token_accuracy": 0.5827347870469093,
+      "step": 6000
+    },
+    {
+      "epoch": 0.044158508665258125,
+      "grad_norm": 1.8545901775360107,
+      "learning_rate": 0.0004779581105593185,
+      "loss": 1.8073,
+      "mean_token_accuracy": 0.5815230685472489,
+      "step": 6500
+    },
+    {
+      "epoch": 0.04755531702412413,
+      "grad_norm": 1.6552695035934448,
+      "learning_rate": 0.00047625970637988545,
+      "loss": 1.7911,
+      "mean_token_accuracy": 0.5853207742571831,
+      "step": 7000
+    },
+    {
+      "epoch": 0.05095212538299014,
+      "grad_norm": 1.9319736957550049,
+      "learning_rate": 0.0004745680958171702,
+      "loss": 1.8075,
+      "mean_token_accuracy": 0.5784689907431603,
+      "step": 7500
+    },
+    {
+      "epoch": 0.054348933741856155,
+      "grad_norm": 1.4338513612747192,
+      "learning_rate": 0.0004728696916377372,
+      "loss": 1.789,
+      "mean_token_accuracy": 0.5835671026110649,
+      "step": 8000
+    },
+    {
+      "epoch": 0.05774574210072216,
+      "grad_norm": 1.7133113145828247,
+      "learning_rate": 0.0004711780810750219,
+      "loss": 1.7669,
+      "mean_token_accuracy": 0.5891133771538735,
+      "step": 8500
+    },
+    {
+      "epoch": 0.06114255045958817,
+      "grad_norm": 1.9302151203155518,
+      "learning_rate": 0.0004694796768955889,
+      "loss": 1.7709,
+      "mean_token_accuracy": 0.5872237432599068,
+      "step": 9000
+    },
+    {
+      "epoch": 0.06453935881845418,
+      "grad_norm": 1.3045645952224731,
+      "learning_rate": 0.0004677880663328736,
+      "loss": 1.7958,
+      "mean_token_accuracy": 0.5849888991117478,
+      "step": 9500
+    },
+    {
+      "epoch": 0.06793616717732019,
+      "grad_norm": 1.1768524646759033,
+      "learning_rate": 0.00046608966215344063,
+      "loss": 1.7645,
+      "mean_token_accuracy": 0.5897379013299942,
+      "step": 10000
+    },
+    {
+      "epoch": 0.0713329755361862,
+      "grad_norm": 1.2476755380630493,
+      "learning_rate": 0.0004643980515907253,
+      "loss": 1.7753,
+      "mean_token_accuracy": 0.5863221107721329,
+      "step": 10500
+    },
+    {
+      "epoch": 0.07472978389505221,
+      "grad_norm": 1.670795202255249,
+      "learning_rate": 0.00046269964741129234,
+      "loss": 1.7885,
+      "mean_token_accuracy": 0.5854759316444397,
+      "step": 11000
+    },
+    {
+      "epoch": 0.07812659225391821,
+      "grad_norm": 1.3658140897750854,
+      "learning_rate": 0.0004610080368485771,
+      "loss": 1.792,
+      "mean_token_accuracy": 0.5836998287439347,
+      "step": 11500
+    },
+    {
+      "epoch": 0.08152340061278422,
+      "grad_norm": 1.7748680114746094,
+      "learning_rate": 0.0004593096326691441,
+      "loss": 1.7749,
+      "mean_token_accuracy": 0.5880664868950843,
+      "step": 12000
+    },
+    {
+      "epoch": 0.08492020897165024,
+      "grad_norm": 2.0673904418945312,
+      "learning_rate": 0.00045761802210642885,
+      "loss": 1.7697,
+      "mean_token_accuracy": 0.5910777195692062,
+      "step": 12500
+    },
+    {
+      "epoch": 0.08831701733051625,
+      "grad_norm": 1.3962881565093994,
+      "learning_rate": 0.0004559196179269958,
+      "loss": 1.763,
+      "mean_token_accuracy": 0.5915324094891549,
+      "step": 13000
+    },
+    {
+      "epoch": 0.09171382568938226,
+      "grad_norm": 1.8353748321533203,
+      "learning_rate": 0.00045422800736428056,
+      "loss": 1.7678,
+      "mean_token_accuracy": 0.5906241601109504,
+      "step": 13500
+    },
+    {
+      "epoch": 0.09511063404824827,
+      "grad_norm": 1.5826489925384521,
+      "learning_rate": 0.0004525296031848475,
+      "loss": 1.7709,
+      "mean_token_accuracy": 0.5858541624546051,
+      "step": 14000
+    },
+    {
+      "epoch": 0.09850744240711427,
+      "grad_norm": 1.4312671422958374,
+      "learning_rate": 0.00045083799262213227,
+      "loss": 1.765,
+      "mean_token_accuracy": 0.5866996909379959,
+      "step": 14500
+    },
+    {
+      "epoch": 0.10190425076598028,
+      "grad_norm": 1.6899515390396118,
+      "learning_rate": 0.00044913958844269923,
+      "loss": 1.769,
+      "mean_token_accuracy": 0.5894079254865646,
+      "step": 15000
+    },
+    {
+      "epoch": 0.10530105912484629,
+      "grad_norm": 1.6989445686340332,
+      "learning_rate": 0.000447447977879984,
+      "loss": 1.761,
+      "mean_token_accuracy": 0.5896116056442261,
+      "step": 15500
+    },
+    {
+      "epoch": 0.10869786748371231,
+      "grad_norm": 1.1391531229019165,
+      "learning_rate": 0.000445749573700551,
+      "loss": 1.7566,
+      "mean_token_accuracy": 0.5887639000415802,
+      "step": 16000
+    },
+    {
+      "epoch": 0.11209467584257832,
+      "grad_norm": 1.3497717380523682,
+      "learning_rate": 0.0004440579631378357,
+      "loss": 1.7834,
+      "mean_token_accuracy": 0.5850634145736694,
+      "step": 16500
+    },
+    {
+      "epoch": 0.11549148420144432,
+      "grad_norm": 1.8933279514312744,
+      "learning_rate": 0.0004423595589584027,
+      "loss": 1.7558,
+      "mean_token_accuracy": 0.5879251533150673,
+      "step": 17000
+    },
+    {
+      "epoch": 0.11888829256031033,
+      "grad_norm": 1.991547703742981,
+      "learning_rate": 0.0004406679483956874,
+      "loss": 1.7611,
+      "mean_token_accuracy": 0.5892675566077232,
+      "step": 17500
+    },
+    {
+      "epoch": 0.12228510091917634,
+      "grad_norm": 1.6691018342971802,
+      "learning_rate": 0.0004389695442162544,
+      "loss": 1.7554,
+      "mean_token_accuracy": 0.587950642824173,
+      "step": 18000
+    },
+    {
+      "epoch": 0.12568190927804235,
+      "grad_norm": 1.443320631980896,
+      "learning_rate": 0.0004372779336535391,
+      "loss": 1.7521,
+      "mean_token_accuracy": 0.5913952052593231,
+      "step": 18500
+    },
+    {
+      "epoch": 0.12907871763690837,
+      "grad_norm": 1.6584292650222778,
+      "learning_rate": 0.0004355795294741061,
+      "loss": 1.7452,
+      "mean_token_accuracy": 0.5926433582901954,
+      "step": 19000
+    },
+    {
+      "epoch": 0.13247552599577436,
+      "grad_norm": 1.787678599357605,
+      "learning_rate": 0.00043388791891139087,
+      "loss": 1.7528,
+      "mean_token_accuracy": 0.5891766239404679,
+      "step": 19500
+    },
+    {
+      "epoch": 0.13587233435464038,
+      "grad_norm": 1.757037878036499,
+      "learning_rate": 0.0004321895147319579,
+      "loss": 1.7529,
+      "mean_token_accuracy": 0.5906364063620567,
+      "step": 20000
+    },
+    {
+      "epoch": 0.13926914271350638,
+      "grad_norm": 1.6813377141952515,
+      "learning_rate": 0.00043049790416924263,
+      "loss": 1.7552,
+      "mean_token_accuracy": 0.5885950907468795,
+      "step": 20500
+    },
+    {
+      "epoch": 0.1426659510723724,
+      "grad_norm": 2.020524501800537,
+      "learning_rate": 0.0004287994999898096,
+      "loss": 1.7478,
+      "mean_token_accuracy": 0.589043869137764,
+      "step": 21000
+    },
+    {
+      "epoch": 0.14606275943123842,
+      "grad_norm": 1.6547284126281738,
+      "learning_rate": 0.00042710788942709434,
+      "loss": 1.7339,
+      "mean_token_accuracy": 0.5956325365304946,
+      "step": 21500
+    },
+    {
+      "epoch": 0.14945956779010441,
+      "grad_norm": 1.7062699794769287,
+      "learning_rate": 0.0004254094852476613,
+      "loss": 1.7433,
+      "mean_token_accuracy": 0.593328341960907,
+      "step": 22000
+    },
+    {
+      "epoch": 0.15285637614897044,
+      "grad_norm": 1.6765140295028687,
+      "learning_rate": 0.00042371787468494605,
+      "loss": 1.7375,
+      "mean_token_accuracy": 0.5927234978675843,
+      "step": 22500
+    },
+    {
+      "epoch": 0.15625318450783643,
+      "grad_norm": 2.00710129737854,
+      "learning_rate": 0.00042201947050551306,
+      "loss": 1.7557,
+      "mean_token_accuracy": 0.588093322455883,
+      "step": 23000
+    },
+    {
+      "epoch": 0.15964999286670245,
+      "grad_norm": 1.338483214378357,
+      "learning_rate": 0.00042032785994279776,
+      "loss": 1.7449,
+      "mean_token_accuracy": 0.5917012639045716,
+      "step": 23500
+    },
+    {
+      "epoch": 0.16304680122556844,
+      "grad_norm": 1.4974863529205322,
+      "learning_rate": 0.00041862945576336477,
+      "loss": 1.7296,
+      "mean_token_accuracy": 0.5941803350448608,
+      "step": 24000
+    },
+    {
+      "epoch": 0.16644360958443447,
+      "grad_norm": 1.5135982036590576,
+      "learning_rate": 0.00041693784520064947,
+      "loss": 1.7291,
+      "mean_token_accuracy": 0.592066936314106,
+      "step": 24500
+    },
+    {
+      "epoch": 0.1698404179433005,
+      "grad_norm": 1.9969964027404785,
+      "learning_rate": 0.0004152394410212165,
+      "loss": 1.7667,
+      "mean_token_accuracy": 0.5857270558476448,
+      "step": 25000
+    },
+    {
+      "epoch": 0.17323722630216648,
+      "grad_norm": 1.9565461874008179,
+      "learning_rate": 0.0004135478304585012,
+      "loss": 1.7682,
+      "mean_token_accuracy": 0.5895871440768242,
+      "step": 25500
+    },
+    {
+      "epoch": 0.1766340346610325,
+      "grad_norm": 1.7885109186172485,
+      "learning_rate": 0.0004118494262790682,
+      "loss": 1.7693,
+      "mean_token_accuracy": 0.5844168608784676,
+      "step": 26000
+    },
+    {
+      "epoch": 0.1800308430198985,
+      "grad_norm": 1.3931702375411987,
+      "learning_rate": 0.0004101578157163529,
+      "loss": 1.7754,
+      "mean_token_accuracy": 0.5874837322831153,
+      "step": 26500
+    },
+    {
+      "epoch": 0.18342765137876452,
+      "grad_norm": 1.3283652067184448,
+      "learning_rate": 0.0004084594115369199,
+      "loss": 1.7504,
+      "mean_token_accuracy": 0.5897262868285179,
+      "step": 27000
+    },
+    {
+      "epoch": 0.1868244597376305,
+      "grad_norm": 1.8210930824279785,
+      "learning_rate": 0.00040676780097420465,
+      "loss": 1.7279,
+      "mean_token_accuracy": 0.5936192185282707,
+      "step": 27500
+    },
+    {
+      "epoch": 0.19022126809649653,
+      "grad_norm": 2.0694313049316406,
+      "learning_rate": 0.0004050693967947716,
+      "loss": 1.7457,
+      "mean_token_accuracy": 0.5926506167650223,
+      "step": 28000
+    },
+    {
+      "epoch": 0.19361807645536255,
+      "grad_norm": 1.5913653373718262,
+      "learning_rate": 0.00040337778623205636,
+      "loss": 1.7331,
+      "mean_token_accuracy": 0.593352241396904,
+      "step": 28500
+    },
+    {
+      "epoch": 0.19701488481422855,
+      "grad_norm": 1.498626708984375,
+      "learning_rate": 0.00040167938205262337,
+      "loss": 1.7368,
+      "mean_token_accuracy": 0.5917426695227623,
+      "step": 29000
+    },
+    {
+      "epoch": 0.20041169317309457,
+      "grad_norm": 1.3690526485443115,
+      "learning_rate": 0.0003999877714899081,
+      "loss": 1.738,
+      "mean_token_accuracy": 0.5922728835344314,
+      "step": 29500
+    },
+    {
+      "epoch": 0.20380850153196056,
+      "grad_norm": 1.6837120056152344,
+      "learning_rate": 0.0003982893673104751,
+      "loss": 1.7264,
+      "mean_token_accuracy": 0.5938475638628006,
+      "step": 30000
+    },
+    {
+      "epoch": 0.20720530989082658,
+      "grad_norm": 1.8007197380065918,
+      "learning_rate": 0.00039659775674775983,
+      "loss": 1.7479,
+      "mean_token_accuracy": 0.5934853712916375,
+      "step": 30500
+    },
+    {
+      "epoch": 0.21060211824969258,
+      "grad_norm": 1.3004428148269653,
+      "learning_rate": 0.00039489935256832684,
+      "loss": 1.7524,
+      "mean_token_accuracy": 0.5907776292562484,
+      "step": 31000
+    },
+    {
+      "epoch": 0.2139989266085586,
+      "grad_norm": 2.0141730308532715,
+      "learning_rate": 0.00039320774200561154,
+      "loss": 1.7271,
+      "mean_token_accuracy": 0.5942207140922546,
+      "step": 31500
+    },
+    {
+      "epoch": 0.21739573496742462,
+      "grad_norm": 1.9760717153549194,
+      "learning_rate": 0.00039150933782617855,
+      "loss": 1.7368,
+      "mean_token_accuracy": 0.5924465500712395,
+      "step": 32000
+    },
+    {
+      "epoch": 0.2207925433262906,
+      "grad_norm": 1.846826195716858,
+      "learning_rate": 0.00038981772726346325,
+      "loss": 1.741,
+      "mean_token_accuracy": 0.5930691966414452,
+      "step": 32500
+    },
+    {
+      "epoch": 0.22418935168515663,
+      "grad_norm": 1.7299907207489014,
+      "learning_rate": 0.00038811932308403026,
+      "loss": 1.7432,
+      "mean_token_accuracy": 0.5922863530516624,
+      "step": 33000
+    },
+    {
+      "epoch": 0.22758616004402263,
+      "grad_norm": 1.3068546056747437,
+      "learning_rate": 0.00038642771252131496,
+      "loss": 1.738,
+      "mean_token_accuracy": 0.591632487475872,
+      "step": 33500
+    },
+    {
+      "epoch": 0.23098296840288865,
+      "grad_norm": 1.4849107265472412,
+      "learning_rate": 0.00038472930834188197,
+      "loss": 1.7485,
+      "mean_token_accuracy": 0.5894864525198936,
+      "step": 34000
+    },
+    {
+      "epoch": 0.23437977676175464,
+      "grad_norm": 1.6784837245941162,
+      "learning_rate": 0.00038303769777916667,
+      "loss": 1.7231,
+      "mean_token_accuracy": 0.5940198531746864,
+      "step": 34500
+    },
+    {
+      "epoch": 0.23777658512062066,
+      "grad_norm": 1.4513353109359741,
+      "learning_rate": 0.0003813392935997337,
+      "loss": 1.7472,
+      "mean_token_accuracy": 0.5893140790462494,
+      "step": 35000
+    },
+    {
+      "epoch": 0.2411733934794867,
+      "grad_norm": 1.5335479974746704,
+      "learning_rate": 0.00037964768303701843,
+      "loss": 1.7408,
+      "mean_token_accuracy": 0.5910349604487419,
+      "step": 35500
+    },
+    {
+      "epoch": 0.24457020183835268,
+      "grad_norm": 1.7906442880630493,
+      "learning_rate": 0.0003779492788575854,
+      "loss": 1.7372,
+      "mean_token_accuracy": 0.591690847337246,
+      "step": 36000
+    },
+    {
+      "epoch": 0.2479670101972187,
+      "grad_norm": 1.6074906587600708,
+      "learning_rate": 0.00037625766829487014,
+      "loss": 1.736,
+      "mean_token_accuracy": 0.5894170867800712,
+      "step": 36500
+    },
+    {
+      "epoch": 0.2513638185560847,
+      "grad_norm": 1.8678852319717407,
+      "learning_rate": 0.00037455926411543715,
+      "loss": 1.7397,
+      "mean_token_accuracy": 0.5938785404562951,
+      "step": 37000
+    },
+    {
+      "epoch": 0.2547606269149507,
+      "grad_norm": 1.4522769451141357,
+      "learning_rate": 0.0003728676535527219,
+      "loss": 1.746,
+      "mean_token_accuracy": 0.5907106329202652,
+      "step": 37500
+    },
+    {
+      "epoch": 0.25815743527381674,
+      "grad_norm": 1.3309742212295532,
+      "learning_rate": 0.00037116924937328886,
+      "loss": 1.7158,
+      "mean_token_accuracy": 0.594115258038044,
+      "step": 38000
+    },
+    {
+      "epoch": 0.2615542436326827,
+      "grad_norm": 1.58572518825531,
+      "learning_rate": 0.0003694776388105736,
+      "loss": 1.733,
+      "mean_token_accuracy": 0.5916760124564171,
+      "step": 38500
+    },
+    {
+      "epoch": 0.2649510519915487,
+      "grad_norm": 2.1666619777679443,
+      "learning_rate": 0.00036778263143949947,
+      "loss": 1.7284,
+      "mean_token_accuracy": 0.5931940126419067,
+      "step": 39000
+    },
+    {
+      "epoch": 0.26834786035041475,
+      "grad_norm": 2.1943438053131104,
+      "learning_rate": 0.0003660842272600665,
+      "loss": 1.7274,
+      "mean_token_accuracy": 0.5921821066141129,
+      "step": 39500
+    },
+    {
+      "epoch": 0.27174466870928077,
+      "grad_norm": 1.406124234199524,
+      "learning_rate": 0.00036438582308063344,
+      "loss": 1.7383,
+      "mean_token_accuracy": 0.5884549335241318,
+      "step": 40000
+    },
+    {
+      "epoch": 0.2751414770681468,
+      "grad_norm": 1.3395054340362549,
+      "learning_rate": 0.00036268741890120045,
+      "loss": 1.7467,
+      "mean_token_accuracy": 0.5905145313739777,
+      "step": 40500
+    },
+    {
+      "epoch": 0.27853828542701276,
+      "grad_norm": 2.2576253414154053,
+      "learning_rate": 0.0003609924115301263,
+      "loss": 1.7405,
+      "mean_token_accuracy": 0.5897097810506821,
+      "step": 41000
+    },
+    {
+      "epoch": 0.2819350937858788,
+      "grad_norm": 1.2508175373077393,
+      "learning_rate": 0.00035929740415905216,
+      "loss": 1.7268,
+      "mean_token_accuracy": 0.5953253656029701,
+      "step": 41500
+    },
+    {
+      "epoch": 0.2853319021447448,
+      "grad_norm": 1.5037624835968018,
+      "learning_rate": 0.00035759899997961917,
+      "loss": 1.7274,
+      "mean_token_accuracy": 0.5924843155145645,
+      "step": 42000
+    },
+    {
+      "epoch": 0.2887287105036108,
+      "grad_norm": 1.236826777458191,
+      "learning_rate": 0.00035590059580018613,
+      "loss": 1.7265,
+      "mean_token_accuracy": 0.5923737421035766,
+      "step": 42500
+    },
+    {
+      "epoch": 0.29212551886247684,
+      "grad_norm": 1.4381422996520996,
+      "learning_rate": 0.00035420219162075314,
+      "loss": 1.7543,
+      "mean_token_accuracy": 0.5863377531170845,
+      "step": 43000
+    },
+    {
+      "epoch": 0.2955223272213428,
+      "grad_norm": 1.2800847291946411,
+      "learning_rate": 0.00035251058105803784,
+      "loss": 1.7313,
+      "mean_token_accuracy": 0.5917149777412415,
+      "step": 43500
+    },
+    {
+      "epoch": 0.29891913558020883,
+      "grad_norm": 1.3125349283218384,
+      "learning_rate": 0.0003508121768786049,
+      "loss": 1.7343,
+      "mean_token_accuracy": 0.5955647248029708,
+      "step": 44000
+    },
+    {
+      "epoch": 0.30231594393907485,
+      "grad_norm": 1.6721398830413818,
+      "learning_rate": 0.0003491205663158896,
+      "loss": 1.7562,
+      "mean_token_accuracy": 0.5855248969197273,
+      "step": 44500
+    },
+    {
+      "epoch": 0.30571275229794087,
+      "grad_norm": 1.6197052001953125,
+      "learning_rate": 0.0003474221621364566,
+      "loss": 1.7283,
+      "mean_token_accuracy": 0.5937187705636024,
+      "step": 45000
+    },
+    {
+      "epoch": 0.30910956065680684,
+      "grad_norm": 1.7727843523025513,
+      "learning_rate": 0.0003457305515737413,
+      "loss": 1.7339,
+      "mean_token_accuracy": 0.593759677529335,
+      "step": 45500
+    },
+    {
+      "epoch": 0.31250636901567286,
+      "grad_norm": 1.6215318441390991,
+      "learning_rate": 0.0003440321473943083,
+      "loss": 1.7106,
+      "mean_token_accuracy": 0.5971788098812103,
+      "step": 46000
+    },
+    {
+      "epoch": 0.3159031773745389,
+      "grad_norm": 1.8316857814788818,
+      "learning_rate": 0.0003423405368315931,
+      "loss": 1.7348,
+      "mean_token_accuracy": 0.5931739917993546,
+      "step": 46500
+    },
+    {
+      "epoch": 0.3192999857334049,
+      "grad_norm": 1.145120620727539,
+      "learning_rate": 0.00034064213265216003,
+      "loss": 1.7056,
+      "mean_token_accuracy": 0.5980896742343903,
+      "step": 47000
+    },
+    {
+      "epoch": 0.3226967940922709,
+      "grad_norm": 1.2346305847167969,
+      "learning_rate": 0.0003389505220894448,
+      "loss": 1.7347,
+      "mean_token_accuracy": 0.5922139519453049,
+      "step": 47500
+    },
+    {
+      "epoch": 0.3260936024511369,
+      "grad_norm": 1.8981019258499146,
+      "learning_rate": 0.00033725211791001174,
+      "loss": 1.7131,
+      "mean_token_accuracy": 0.59793397295475,
+      "step": 48000
+    },
+    {
+      "epoch": 0.3294904108100029,
+      "grad_norm": 1.204249620437622,
+      "learning_rate": 0.00033556390415565533,
+      "loss": 1.7218,
+      "mean_token_accuracy": 0.5927057997584343,
+      "step": 48500
+    },
+    {
+      "epoch": 0.33288721916886893,
+      "grad_norm": 1.1320068836212158,
+      "learning_rate": 0.00033386549997622235,
+      "loss": 1.7449,
+      "mean_token_accuracy": 0.5909441923499107,
+      "step": 49000
+    },
+    {
+      "epoch": 0.33628402752773495,
+      "grad_norm": 1.3632432222366333,
+      "learning_rate": 0.00033216709579678936,
+      "loss": 1.7346,
+      "mean_token_accuracy": 0.5936996278166771,
+      "step": 49500
+    },
+    {
+      "epoch": 0.339680835886601,
+      "grad_norm": 1.7296600341796875,
+      "learning_rate": 0.0003304720884257152,
+      "loss": 1.7122,
+      "mean_token_accuracy": 0.5955489371418953,
+      "step": 50000
+    },
+    {
+      "epoch": 0.34307764424546694,
+      "grad_norm": 1.5467466115951538,
+      "learning_rate": 0.00032877708105464107,
+      "loss": 1.712,
+      "mean_token_accuracy": 0.5948838356733323,
+      "step": 50500
+    },
+    {
+      "epoch": 0.34647445260433296,
+      "grad_norm": 1.757464051246643,
+      "learning_rate": 0.000327078676875208,
+      "loss": 1.7187,
+      "mean_token_accuracy": 0.5950654003024102,
+      "step": 51000
+    },
+    {
+      "epoch": 0.349871260963199,
+      "grad_norm": 1.5410348176956177,
+      "learning_rate": 0.0003253802726957751,
+      "loss": 1.7029,
+      "mean_token_accuracy": 0.5985295962691307,
+      "step": 51500
+    },
+    {
+      "epoch": 0.353268069322065,
+      "grad_norm": 1.2318569421768188,
+      "learning_rate": 0.00032368186851634205,
+      "loss": 1.734,
+      "mean_token_accuracy": 0.5905957593917847,
+      "step": 52000
+    },
+    {
+      "epoch": 0.35666487768093097,
+      "grad_norm": 1.3083893060684204,
+      "learning_rate": 0.0003219902579536268,
+      "loss": 1.7285,
+      "mean_token_accuracy": 0.5927301960587501,
+      "step": 52500
+    },
+    {
+      "epoch": 0.360061686039797,
+      "grad_norm": 1.3331634998321533,
+      "learning_rate": 0.0003202918537741938,
+      "loss": 1.7345,
+      "mean_token_accuracy": 0.5932314485311508,
+      "step": 53000
+    },
+    {
+      "epoch": 0.363458494398663,
+      "grad_norm": 1.4945528507232666,
+      "learning_rate": 0.0003186002432114785,
+      "loss": 1.7306,
+      "mean_token_accuracy": 0.5918755218982696,
+      "step": 53500
+    },
+    {
+      "epoch": 0.36685530275752903,
+      "grad_norm": 1.6073297262191772,
+      "learning_rate": 0.0003169018390320455,
+      "loss": 1.699,
+      "mean_token_accuracy": 0.5984663163423538,
+      "step": 54000
+    },
+    {
+      "epoch": 0.37025211111639506,
+      "grad_norm": 1.6728241443634033,
+      "learning_rate": 0.0003152102284693302,
+      "loss": 1.7029,
+      "mean_token_accuracy": 0.5983245507478714,
+      "step": 54500
+    },
+    {
+      "epoch": 0.373648919475261,
+      "grad_norm": 1.244017481803894,
+      "learning_rate": 0.00031351182428989723,
+      "loss": 1.7171,
+      "mean_token_accuracy": 0.5940723571181298,
+      "step": 55000
+    },
+    {
+      "epoch": 0.37704572783412704,
+      "grad_norm": 1.6529678106307983,
+      "learning_rate": 0.00031182021372718193,
+      "loss": 1.7247,
+      "mean_token_accuracy": 0.5914634907245636,
+      "step": 55500
+    },
+    {
+      "epoch": 0.38044253619299306,
+      "grad_norm": 1.12068510055542,
+      "learning_rate": 0.00031012180954774894,
+      "loss": 1.7282,
+      "mean_token_accuracy": 0.5951982579231262,
+      "step": 56000
+    },
+    {
+      "epoch": 0.3838393445518591,
+      "grad_norm": 1.516836404800415,
+      "learning_rate": 0.00030843359579339253,
+      "loss": 1.7246,
+      "mean_token_accuracy": 0.5960168350338936,
+      "step": 56500
+    },
+    {
+      "epoch": 0.3872361529107251,
+      "grad_norm": 1.2628341913223267,
+      "learning_rate": 0.00030673519161395955,
+      "loss": 1.7475,
+      "mean_token_accuracy": 0.5910480327606201,
+      "step": 57000
+    },
+    {
+      "epoch": 0.3906329612695911,
+      "grad_norm": 2.0328640937805176,
+      "learning_rate": 0.0003050367874345265,
+      "loss": 1.7238,
+      "mean_token_accuracy": 0.5961861664056778,
+      "step": 57500
+    },
+    {
+      "epoch": 0.3940297696284571,
+      "grad_norm": 1.4986746311187744,
+      "learning_rate": 0.0003033383832550935,
+      "loss": 1.7174,
+      "mean_token_accuracy": 0.5933957831859589,
+      "step": 58000
+    },
+    {
+      "epoch": 0.3974265779873231,
+      "grad_norm": 1.7555750608444214,
+      "learning_rate": 0.0003016467726923782,
+      "loss": 1.7153,
+      "mean_token_accuracy": 0.5939786534309387,
+      "step": 58500
+    },
+    {
+      "epoch": 0.40082338634618914,
+      "grad_norm": 2.061699390411377,
+      "learning_rate": 0.00029995176532130407,
+      "loss": 1.7122,
+      "mean_token_accuracy": 0.5955729194283486,
+      "step": 59000
+    },
+    {
+      "epoch": 0.4042201947050551,
+      "grad_norm": 1.5404950380325317,
+      "learning_rate": 0.00029825336114187113,
+      "loss": 1.7236,
+      "mean_token_accuracy": 0.5967325463294982,
+      "step": 59500
+    },
+    {
+      "epoch": 0.4076170030639211,
+      "grad_norm": 1.7109909057617188,
+      "learning_rate": 0.0002965549569624381,
+      "loss": 1.7503,
+      "mean_token_accuracy": 0.5910088227987289,
+      "step": 60000
+    },
+    {
+      "epoch": 0.41101381142278715,
+      "grad_norm": 1.6398552656173706,
+      "learning_rate": 0.000294859949591364,
+      "loss": 1.7196,
+      "mean_token_accuracy": 0.5942435787916184,
+      "step": 60500
+    },
+    {
+      "epoch": 0.41441061978165317,
+      "grad_norm": 1.974751353263855,
+      "learning_rate": 0.00029316154541193096,
+      "loss": 1.7271,
+      "mean_token_accuracy": 0.5930974147915841,
+      "step": 61000
+    },
+    {
+      "epoch": 0.4178074281405192,
+      "grad_norm": 1.449018955230713,
+      "learning_rate": 0.0002914699348492157,
+      "loss": 1.7031,
+      "mean_token_accuracy": 0.5974128057360649,
+      "step": 61500
+    },
+    {
+      "epoch": 0.42120423649938515,
+      "grad_norm": 1.4091970920562744,
+      "learning_rate": 0.00028977153066978267,
+      "loss": 1.705,
+      "mean_token_accuracy": 0.597780381500721,
+      "step": 62000
+    },
+    {
+      "epoch": 0.4246010448582512,
+      "grad_norm": 1.491416573524475,
+      "learning_rate": 0.0002880799201070674,
+      "loss": 1.7088,
+      "mean_token_accuracy": 0.5958248073458672,
+      "step": 62500
+    },
+    {
+      "epoch": 0.4279978532171172,
+      "grad_norm": 1.1961016654968262,
+      "learning_rate": 0.0002863815159276344,
+      "loss": 1.7192,
+      "mean_token_accuracy": 0.5927025334239006,
+      "step": 63000
+    },
+    {
+      "epoch": 0.4313946615759832,
+      "grad_norm": 1.0604102611541748,
+      "learning_rate": 0.00028468990536491913,
+      "loss": 1.7503,
+      "mean_token_accuracy": 0.589413387954235,
+      "step": 63500
+    },
+    {
+      "epoch": 0.43479146993484924,
+      "grad_norm": 1.3752529621124268,
+      "learning_rate": 0.0002829915011854861,
+      "loss": 1.7176,
+      "mean_token_accuracy": 0.5928765575289726,
+      "step": 64000
+    },
+    {
+      "epoch": 0.4381882782937152,
+      "grad_norm": 1.339768886566162,
+      "learning_rate": 0.00028129989062277084,
+      "loss": 1.7227,
+      "mean_token_accuracy": 0.5919658952355384,
+      "step": 64500
+    },
+    {
+      "epoch": 0.4415850866525812,
+      "grad_norm": 1.4945833683013916,
+      "learning_rate": 0.00027960148644333785,
+      "loss": 1.7304,
+      "mean_token_accuracy": 0.5910812721252442,
+      "step": 65000
+    },
+    {
+      "epoch": 0.44498189501144725,
+      "grad_norm": 1.4786356687545776,
+      "learning_rate": 0.00027790987588062255,
+      "loss": 1.7203,
+      "mean_token_accuracy": 0.5954633237123489,
+      "step": 65500
+    },
+    {
+      "epoch": 0.44837870337031327,
+      "grad_norm": 1.5545837879180908,
+      "learning_rate": 0.00027621147170118956,
+      "loss": 1.6998,
+      "mean_token_accuracy": 0.5979457493424416,
+      "step": 66000
+    },
+    {
+      "epoch": 0.45177551172917924,
+      "grad_norm": 1.7460355758666992,
+      "learning_rate": 0.00027452325794683315,
+      "loss": 1.7144,
+      "mean_token_accuracy": 0.5964882708787919,
+      "step": 66500
+    },
+    {
+      "epoch": 0.45517232008804526,
+      "grad_norm": 1.800485372543335,
+      "learning_rate": 0.00027282485376740016,
+      "loss": 1.7256,
+      "mean_token_accuracy": 0.5932715454101563,
+      "step": 67000
+    },
+    {
+      "epoch": 0.4585691284469113,
+      "grad_norm": 1.8166261911392212,
+      "learning_rate": 0.0002711264495879672,
+      "loss": 1.7173,
+      "mean_token_accuracy": 0.5953007768392563,
+      "step": 67500
+    },
+    {
+      "epoch": 0.4619659368057773,
+      "grad_norm": 1.3276996612548828,
+      "learning_rate": 0.0002694280454085342,
+      "loss": 1.7048,
+      "mean_token_accuracy": 0.5957981157302856,
+      "step": 68000
+    },
+    {
+      "epoch": 0.4653627451646433,
+      "grad_norm": 1.5296154022216797,
+      "learning_rate": 0.0002677364348458189,
+      "loss": 1.6992,
+      "mean_token_accuracy": 0.5997445729374885,
+      "step": 68500
+    },
+    {
+      "epoch": 0.4687595535235093,
+      "grad_norm": 1.8093739748001099,
+      "learning_rate": 0.0002660380306663859,
+      "loss": 1.7024,
+      "mean_token_accuracy": 0.5984397698640823,
+      "step": 69000
+    },
+    {
+      "epoch": 0.4721563618823753,
+      "grad_norm": 1.6000614166259766,
+      "learning_rate": 0.00026433962648695286,
+      "loss": 1.6995,
+      "mean_token_accuracy": 0.5959528124332428,
+      "step": 69500
+    },
+    {
+      "epoch": 0.47555317024124133,
+      "grad_norm": 2.142077684402466,
+      "learning_rate": 0.00026264122230751987,
+      "loss": 1.7166,
+      "mean_token_accuracy": 0.5951734070181847,
+      "step": 70000
+    },
+    {
+      "epoch": 0.47894997860010735,
+      "grad_norm": 1.944488525390625,
+      "learning_rate": 0.0002609462149364457,
+      "loss": 1.6911,
+      "mean_token_accuracy": 0.5987489570379257,
+      "step": 70500
+    },
+    {
+      "epoch": 0.4823467869589734,
+      "grad_norm": 1.4645779132843018,
+      "learning_rate": 0.0002592478107570127,
+      "loss": 1.7089,
+      "mean_token_accuracy": 0.5985424500107765,
+      "step": 71000
+    },
+    {
+      "epoch": 0.48574359531783934,
+      "grad_norm": 1.5057988166809082,
+      "learning_rate": 0.00025755620019429743,
+      "loss": 1.7265,
+      "mean_token_accuracy": 0.5922452852129936,
+      "step": 71500
+    },
+    {
+      "epoch": 0.48914040367670536,
+      "grad_norm": 1.4694526195526123,
+      "learning_rate": 0.00025585779601486444,
+      "loss": 1.701,
+      "mean_token_accuracy": 0.5997735613584518,
+      "step": 72000
+    },
+    {
+      "epoch": 0.4925372120355714,
+      "grad_norm": 1.502824068069458,
+      "learning_rate": 0.00025416618545214914,
+      "loss": 1.6909,
+      "mean_token_accuracy": 0.60144336104393,
+      "step": 72500
+    },
+    {
+      "epoch": 0.4959340203944374,
+      "grad_norm": 1.43698251247406,
+      "learning_rate": 0.00025246778127271615,
+      "loss": 1.7215,
+      "mean_token_accuracy": 0.592339691400528,
+      "step": 73000
+    },
+    {
+      "epoch": 0.49933082875330337,
+      "grad_norm": 1.2727726697921753,
+      "learning_rate": 0.00025077617071000085,
+      "loss": 1.7169,
+      "mean_token_accuracy": 0.5964000345468521,
+      "step": 73500
+    },
+    {
+      "epoch": 0.5027276371121694,
+      "grad_norm": 1.7707308530807495,
+      "learning_rate": 0.00024907776653056786,
+      "loss": 1.7242,
+      "mean_token_accuracy": 0.5944222602248191,
+      "step": 74000
+    },
+    {
+      "epoch": 0.5061244454710354,
+      "grad_norm": 1.5148441791534424,
+      "learning_rate": 0.0002473861559678526,
+      "loss": 1.7261,
+      "mean_token_accuracy": 0.5926074689626694,
+      "step": 74500
+    },
+    {
+      "epoch": 0.5095212538299014,
+      "grad_norm": 1.513931393623352,
+      "learning_rate": 0.00024568775178841957,
+      "loss": 1.6951,
+      "mean_token_accuracy": 0.5987407351732255,
+      "step": 75000
+    },
+    {
+      "epoch": 0.5129180621887675,
+      "grad_norm": 1.5299018621444702,
+      "learning_rate": 0.0002439995380340632,
+      "loss": 1.6984,
+      "mean_token_accuracy": 0.5971011264324189,
+      "step": 75500
+    },
+    {
+      "epoch": 0.5163148705476335,
+      "grad_norm": 1.2403771877288818,
+      "learning_rate": 0.0002423011338546302,
+      "loss": 1.7133,
+      "mean_token_accuracy": 0.5969100252985954,
+      "step": 76000
+    },
+    {
+      "epoch": 0.5197116789064995,
+      "grad_norm": 1.163865089416504,
+      "learning_rate": 0.0002406027296751972,
+      "loss": 1.7216,
+      "mean_token_accuracy": 0.5946598027348519,
+      "step": 76500
+    },
+    {
+      "epoch": 0.5231084872653654,
+      "grad_norm": 1.1987580060958862,
+      "learning_rate": 0.00023890772230412304,
+      "loss": 1.7114,
+      "mean_token_accuracy": 0.5971543624997139,
+      "step": 77000
+    },
+    {
+      "epoch": 0.5265052956242314,
+      "grad_norm": 1.5219857692718506,
+      "learning_rate": 0.00023720931812469005,
+      "loss": 1.7268,
+      "mean_token_accuracy": 0.5944567822813988,
+      "step": 77500
+    },
+    {
+      "epoch": 0.5299021039830974,
+      "grad_norm": 1.7447556257247925,
+      "learning_rate": 0.00023551770756197478,
+      "loss": 1.7067,
+      "mean_token_accuracy": 0.5983748460412025,
+      "step": 78000
+    },
+    {
+      "epoch": 0.5332989123419635,
+      "grad_norm": 1.5807901620864868,
+      "learning_rate": 0.00023381930338254176,
+      "loss": 1.7022,
+      "mean_token_accuracy": 0.596363877415657,
+      "step": 78500
+    },
+    {
+      "epoch": 0.5366957207008295,
+      "grad_norm": 1.2615615129470825,
+      "learning_rate": 0.00023213108962818536,
+      "loss": 1.7074,
+      "mean_token_accuracy": 0.5990927094221115,
+      "step": 79000
+    },
+    {
+      "epoch": 0.5400925290596955,
+      "grad_norm": 1.3129615783691406,
+      "learning_rate": 0.00023043268544875234,
+      "loss": 1.6992,
+      "mean_token_accuracy": 0.5990150774717331,
+      "step": 79500
+    },
+    {
+      "epoch": 0.5434893374185615,
+      "grad_norm": 1.70474112033844,
+      "learning_rate": 0.00022873428126931935,
+      "loss": 1.7002,
+      "mean_token_accuracy": 0.60052221429348,
+      "step": 80000
+    },
+    {
+      "epoch": 0.5468861457774276,
+      "grad_norm": 1.5991852283477783,
+      "learning_rate": 0.00022703587708988634,
+      "loss": 1.7147,
+      "mean_token_accuracy": 0.5944349249601364,
+      "step": 80500
+    },
+    {
+      "epoch": 0.5502829541362936,
+      "grad_norm": 1.4010807275772095,
+      "learning_rate": 0.00022534426652717106,
+      "loss": 1.7051,
+      "mean_token_accuracy": 0.5958114340305328,
+      "step": 81000
+    },
+    {
+      "epoch": 0.5536797624951596,
+      "grad_norm": 1.8371765613555908,
+      "learning_rate": 0.00022364586234773808,
+      "loss": 1.7214,
+      "mean_token_accuracy": 0.5937463521361351,
+      "step": 81500
+    },
+    {
+      "epoch": 0.5570765708540255,
+      "grad_norm": 1.4573875665664673,
+      "learning_rate": 0.0002219542517850228,
+      "loss": 1.722,
+      "mean_token_accuracy": 0.5933739170432091,
+      "step": 82000
+    },
+    {
+      "epoch": 0.5604733792128915,
+      "grad_norm": 1.379260778427124,
+      "learning_rate": 0.00022025584760558978,
+      "loss": 1.719,
+      "mean_token_accuracy": 0.5941837483644485,
+      "step": 82500
+    },
+    {
+      "epoch": 0.5638701875717576,
+      "grad_norm": 1.7039533853530884,
+      "learning_rate": 0.00021856084023451564,
+      "loss": 1.6781,
+      "mean_token_accuracy": 0.5997574129104615,
+      "step": 83000
+    },
+    {
+      "epoch": 0.5672669959306236,
+      "grad_norm": 1.4409757852554321,
+      "learning_rate": 0.00021686243605508265,
+      "loss": 1.7089,
+      "mean_token_accuracy": 0.5973666083812713,
+      "step": 83500
+    },
+    {
+      "epoch": 0.5706638042894896,
+      "grad_norm": 1.468658685684204,
+      "learning_rate": 0.00021516403187564964,
+      "loss": 1.6952,
+      "mean_token_accuracy": 0.5997344598770141,
+      "step": 84000
+    },
+    {
+      "epoch": 0.5740606126483556,
+      "grad_norm": 1.6600291728973389,
+      "learning_rate": 0.00021346562769621662,
+      "loss": 1.7027,
+      "mean_token_accuracy": 0.5958540776371956,
+      "step": 84500
+    },
+    {
+      "epoch": 0.5774574210072216,
+      "grad_norm": 1.8872114419937134,
+      "learning_rate": 0.00021177401713350137,
+      "loss": 1.7216,
+      "mean_token_accuracy": 0.5948761952519417,
+      "step": 85000
+    },
+    {
+      "epoch": 0.5808542293660877,
+      "grad_norm": 2.2531039714813232,
+      "learning_rate": 0.00021007561295406836,
+      "loss": 1.6948,
+      "mean_token_accuracy": 0.6000947986841202,
+      "step": 85500
+    },
+    {
+      "epoch": 0.5842510377249537,
+      "grad_norm": 1.4668636322021484,
+      "learning_rate": 0.00020838400239135308,
+      "loss": 1.7158,
+      "mean_token_accuracy": 0.5976604241728782,
+      "step": 86000
+    },
+    {
+      "epoch": 0.5876478460838196,
+      "grad_norm": 1.9422500133514404,
+      "learning_rate": 0.0002066855982119201,
+      "loss": 1.7247,
+      "mean_token_accuracy": 0.5928294175267219,
+      "step": 86500
+    },
+    {
+      "epoch": 0.5910446544426856,
+      "grad_norm": 1.4086047410964966,
+      "learning_rate": 0.00020499398764920482,
+      "loss": 1.7333,
+      "mean_token_accuracy": 0.5915305185317993,
+      "step": 87000
+    },
+    {
+      "epoch": 0.5944414628015516,
+      "grad_norm": 1.6210030317306519,
+      "learning_rate": 0.0002032955834697718,
+      "loss": 1.6975,
+      "mean_token_accuracy": 0.598506959438324,
+      "step": 87500
+    },
+    {
+      "epoch": 0.5978382711604177,
+      "grad_norm": 1.4868794679641724,
+      "learning_rate": 0.00020160397290705653,
+      "loss": 1.7265,
+      "mean_token_accuracy": 0.5912136439681053,
+      "step": 88000
+    },
+    {
+      "epoch": 0.6012350795192837,
+      "grad_norm": 1.7365410327911377,
+      "learning_rate": 0.00019990556872762354,
+      "loss": 1.7056,
+      "mean_token_accuracy": 0.5971870743632317,
+      "step": 88500
+    },
+    {
+      "epoch": 0.6046318878781497,
+      "grad_norm": 1.3171368837356567,
+      "learning_rate": 0.00019821395816490826,
+      "loss": 1.7021,
+      "mean_token_accuracy": 0.5960493548512459,
+      "step": 89000
+    },
+    {
+      "epoch": 0.6080286962370157,
+      "grad_norm": 1.2266417741775513,
+      "learning_rate": 0.00019651555398547525,
+      "loss": 1.692,
+      "mean_token_accuracy": 0.5992095020413398,
+      "step": 89500
+    },
+    {
+      "epoch": 0.6114255045958817,
+      "grad_norm": 1.648574948310852,
+      "learning_rate": 0.00019482394342275997,
+      "loss": 1.7127,
+      "mean_token_accuracy": 0.5936791259646416,
+      "step": 90000
+    },
+    {
+      "epoch": 0.6148223129547478,
+      "grad_norm": 1.2666032314300537,
+      "learning_rate": 0.00019312553924332698,
+      "loss": 1.714,
+      "mean_token_accuracy": 0.5962395486235619,
+      "step": 90500
+    },
+    {
+      "epoch": 0.6182191213136137,
+      "grad_norm": 1.6018437147140503,
+      "learning_rate": 0.0001914339286806117,
+      "loss": 1.7324,
+      "mean_token_accuracy": 0.593222697019577,
+      "step": 91000
+    },
+    {
+      "epoch": 0.6216159296724797,
+      "grad_norm": 1.6640033721923828,
+      "learning_rate": 0.00018973892130953756,
+      "loss": 1.6999,
+      "mean_token_accuracy": 0.5964140148162842,
+      "step": 91500
+    },
+    {
+      "epoch": 0.6250127380313457,
+      "grad_norm": 1.5008821487426758,
+      "learning_rate": 0.00018804051713010455,
+      "loss": 1.6975,
+      "mean_token_accuracy": 0.5984971014261246,
+      "step": 92000
+    },
+    {
+      "epoch": 0.6284095463902117,
+      "grad_norm": 1.6151808500289917,
+      "learning_rate": 0.00018634211295067156,
+      "loss": 1.7232,
+      "mean_token_accuracy": 0.593654693365097,
+      "step": 92500
+    },
+    {
+      "epoch": 0.6318063547490778,
+      "grad_norm": 1.8144086599349976,
+      "learning_rate": 0.00018464370877123855,
+      "loss": 1.7106,
+      "mean_token_accuracy": 0.5946467993855477,
+      "step": 93000
+    },
+    {
+      "epoch": 0.6352031631079438,
+      "grad_norm": 1.2094484567642212,
+      "learning_rate": 0.00018295209820852327,
+      "loss": 1.7074,
+      "mean_token_accuracy": 0.5950092726945877,
+      "step": 93500
+    },
+    {
+      "epoch": 0.6385999714668098,
+      "grad_norm": 1.7833865880966187,
+      "learning_rate": 0.00018125369402909028,
+      "loss": 1.6829,
+      "mean_token_accuracy": 0.604508092880249,
+      "step": 94000
+    },
+    {
+      "epoch": 0.6419967798256758,
+      "grad_norm": 1.4266043901443481,
+      "learning_rate": 0.000179562083466375,
+      "loss": 1.7049,
+      "mean_token_accuracy": 0.5994778836369514,
+      "step": 94500
+    },
+    {
+      "epoch": 0.6453935881845418,
+      "grad_norm": 1.3218567371368408,
+      "learning_rate": 0.000177863679286942,
+      "loss": 1.718,
+      "mean_token_accuracy": 0.594626106619835,
+      "step": 95000
+    },
+    {
+      "epoch": 0.6487903965434079,
+      "grad_norm": 1.3818122148513794,
+      "learning_rate": 0.00017617206872422671,
+      "loss": 1.7132,
+      "mean_token_accuracy": 0.5952843418717384,
+      "step": 95500
+    },
+    {
+      "epoch": 0.6521872049022738,
+      "grad_norm": 1.7150118350982666,
+      "learning_rate": 0.00017447366454479373,
+      "loss": 1.7035,
+      "mean_token_accuracy": 0.5954197651743889,
+      "step": 96000
+    },
+    {
+      "epoch": 0.6555840132611398,
+      "grad_norm": 1.8039283752441406,
+      "learning_rate": 0.00017278205398207845,
+      "loss": 1.6804,
+      "mean_token_accuracy": 0.6031543609499931,
+      "step": 96500
+    },
+    {
+      "epoch": 0.6589808216200058,
+      "grad_norm": 2.61078143119812,
+      "learning_rate": 0.00017108364980264544,
+      "loss": 1.7241,
+      "mean_token_accuracy": 0.5923587481379509,
+      "step": 97000
+    },
+    {
+      "epoch": 0.6623776299788718,
+      "grad_norm": 1.8881034851074219,
+      "learning_rate": 0.00016939543604828903,
+      "loss": 1.6988,
+      "mean_token_accuracy": 0.5967677654027939,
+      "step": 97500
+    },
+    {
+      "epoch": 0.6657744383377379,
+      "grad_norm": 1.6494427919387817,
+      "learning_rate": 0.00016769703186885601,
+      "loss": 1.684,
+      "mean_token_accuracy": 0.6023535789251327,
+      "step": 98000
+    },
+    {
+      "epoch": 0.6691712466966039,
+      "grad_norm": 2.0881924629211426,
+      "learning_rate": 0.00016599862768942303,
+      "loss": 1.7006,
+      "mean_token_accuracy": 0.5999556155800819,
+      "step": 98500
+    },
+    {
+      "epoch": 0.6725680550554699,
+      "grad_norm": 1.3696866035461426,
+      "learning_rate": 0.00016430022350999,
+      "loss": 1.7175,
+      "mean_token_accuracy": 0.5950366785526275,
+      "step": 99000
+    },
+    {
+      "epoch": 0.6759648634143359,
+      "grad_norm": 1.5105955600738525,
+      "learning_rate": 0.00016260861294727474,
+      "loss": 1.7096,
+      "mean_token_accuracy": 0.5943602663874626,
+      "step": 99500
+    },
+    {
+      "epoch": 0.679361671773202,
+      "grad_norm": 1.633851408958435,
+      "learning_rate": 0.00016091020876784175,
+      "loss": 1.706,
+      "mean_token_accuracy": 0.5979482631683349,
+      "step": 100000
+    },
+    {
+      "epoch": 0.6827584801320679,
+      "grad_norm": 1.245695948600769,
+      "learning_rate": 0.00015921859820512647,
+      "loss": 1.7146,
+      "mean_token_accuracy": 0.5953260178565979,
+      "step": 100500
+    },
+    {
+      "epoch": 0.6861552884909339,
+      "grad_norm": 1.507904052734375,
+      "learning_rate": 0.00015752019402569346,
+      "loss": 1.6998,
+      "mean_token_accuracy": 0.5975649612545967,
+      "step": 101000
+    },
+    {
+      "epoch": 0.6895520968497999,
+      "grad_norm": 1.4007651805877686,
+      "learning_rate": 0.00015582858346297818,
+      "loss": 1.6847,
+      "mean_token_accuracy": 0.597107557952404,
+      "step": 101500
+    },
+    {
+      "epoch": 0.6929489052086659,
+      "grad_norm": 2.0587270259857178,
+      "learning_rate": 0.0001541301792835452,
+      "loss": 1.726,
+      "mean_token_accuracy": 0.5939525011777878,
+      "step": 102000
+    },
+    {
+      "epoch": 0.6963457135675319,
+      "grad_norm": 1.8779165744781494,
+      "learning_rate": 0.00015243856872082992,
+      "loss": 1.7059,
+      "mean_token_accuracy": 0.5953717207908631,
+      "step": 102500
+    },
+    {
+      "epoch": 0.699742521926398,
+      "grad_norm": 1.5524628162384033,
+      "learning_rate": 0.0001507401645413969,
+      "loss": 1.7098,
+      "mean_token_accuracy": 0.5971698425412179,
+      "step": 103000
+    },
+    {
+      "epoch": 0.703139330285264,
+      "grad_norm": 1.5052322149276733,
+      "learning_rate": 0.00014904855397868163,
+      "loss": 1.7005,
+      "mean_token_accuracy": 0.5954086900949478,
+      "step": 103500
+    },
+    {
+      "epoch": 0.70653613864413,
+      "grad_norm": 1.553138017654419,
+      "learning_rate": 0.00014735014979924864,
+      "loss": 1.6897,
+      "mean_token_accuracy": 0.5997277453541756,
+      "step": 104000
+    },
+    {
+      "epoch": 0.709932947002996,
+      "grad_norm": 1.344943881034851,
+      "learning_rate": 0.00014565853923653336,
+      "loss": 1.697,
+      "mean_token_accuracy": 0.5969586735367775,
+      "step": 104500
+    },
+    {
+      "epoch": 0.7133297553618619,
+      "grad_norm": 1.2146844863891602,
+      "learning_rate": 0.00014396013505710035,
+      "loss": 1.7018,
+      "mean_token_accuracy": 0.5975850567817688,
+      "step": 105000
+    },
+    {
+      "epoch": 0.716726563720728,
+      "grad_norm": 1.6853300333023071,
+      "learning_rate": 0.00014226852449438507,
+      "loss": 1.7151,
+      "mean_token_accuracy": 0.5967495339512825,
+      "step": 105500
+    },
+    {
+      "epoch": 0.720123372079594,
+      "grad_norm": 1.525900959968567,
+      "learning_rate": 0.00014057012031495206,
+      "loss": 1.7099,
+      "mean_token_accuracy": 0.5934486684799194,
+      "step": 106000
+    },
+    {
+      "epoch": 0.72352018043846,
+      "grad_norm": 1.8473764657974243,
+      "learning_rate": 0.0001388785097522368,
+      "loss": 1.6943,
+      "mean_token_accuracy": 0.5962625173926354,
+      "step": 106500
+    },
+    {
+      "epoch": 0.726916988797326,
+      "grad_norm": 1.8339717388153076,
+      "learning_rate": 0.00013718350238116266,
+      "loss": 1.696,
+      "mean_token_accuracy": 0.5971350433826447,
+      "step": 107000
+    },
+    {
+      "epoch": 0.730313797156192,
+      "grad_norm": 1.4246865510940552,
+      "learning_rate": 0.00013548509820172967,
+      "loss": 1.7098,
+      "mean_token_accuracy": 0.5951427467465401,
+      "step": 107500
+    },
+    {
+      "epoch": 0.7337106055150581,
+      "grad_norm": 1.863608717918396,
+      "learning_rate": 0.00013378669402229666,
+      "loss": 1.6962,
+      "mean_token_accuracy": 0.600303438603878,
+      "step": 108000
+    },
+    {
+      "epoch": 0.7371074138739241,
+      "grad_norm": 1.392205834388733,
+      "learning_rate": 0.00013208828984286364,
+      "loss": 1.721,
+      "mean_token_accuracy": 0.5922597636580467,
+      "step": 108500
+    },
+    {
+      "epoch": 0.7405042222327901,
+      "grad_norm": 1.7147854566574097,
+      "learning_rate": 0.00013039667928014837,
+      "loss": 1.6914,
+      "mean_token_accuracy": 0.5969484311938286,
+      "step": 109000
+    },
+    {
+      "epoch": 0.7439010305916561,
+      "grad_norm": 1.7911548614501953,
+      "learning_rate": 0.00012869827510071538,
+      "loss": 1.687,
+      "mean_token_accuracy": 0.5990350655913353,
+      "step": 109500
+    },
+    {
+      "epoch": 0.747297838950522,
+      "grad_norm": 1.5777207612991333,
+      "learning_rate": 0.0001270066645380001,
+      "loss": 1.6877,
+      "mean_token_accuracy": 0.599417865395546,
+      "step": 110000
+    },
+    {
+      "epoch": 0.7506946473093881,
+      "grad_norm": 1.9362062215805054,
+      "learning_rate": 0.0001253082603585671,
+      "loss": 1.7135,
+      "mean_token_accuracy": 0.5962401946783066,
+      "step": 110500
+    },
+    {
+      "epoch": 0.7540914556682541,
+      "grad_norm": 1.4036149978637695,
+      "learning_rate": 0.00012361664979585181,
+      "loss": 1.7054,
+      "mean_token_accuracy": 0.597572597026825,
+      "step": 111000
+    },
+    {
+      "epoch": 0.7574882640271201,
+      "grad_norm": 1.4795775413513184,
+      "learning_rate": 0.00012191824561641881,
+      "loss": 1.6865,
+      "mean_token_accuracy": 0.6025166733264923,
+      "step": 111500
+    },
+    {
+      "epoch": 0.7608850723859861,
+      "grad_norm": 1.5477979183197021,
+      "learning_rate": 0.00012022663505370354,
+      "loss": 1.6811,
+      "mean_token_accuracy": 0.602429338991642,
+      "step": 112000
+    },
+    {
+      "epoch": 0.7642818807448521,
+      "grad_norm": 1.2410552501678467,
+      "learning_rate": 0.00011852823087427054,
+      "loss": 1.7032,
+      "mean_token_accuracy": 0.5962775309085846,
+      "step": 112500
+    },
+    {
+      "epoch": 0.7676786891037182,
+      "grad_norm": 1.4975206851959229,
+      "learning_rate": 0.00011683662031155526,
+      "loss": 1.6824,
+      "mean_token_accuracy": 0.5965480819940567,
+      "step": 113000
+    },
+    {
+      "epoch": 0.7710754974625842,
+      "grad_norm": 1.1648781299591064,
+      "learning_rate": 0.00011513821613212226,
+      "loss": 1.6886,
+      "mean_token_accuracy": 0.6017087162137031,
+      "step": 113500
+    },
+    {
+      "epoch": 0.7744723058214502,
+      "grad_norm": 1.6594377756118774,
+      "learning_rate": 0.00011344660556940698,
+      "loss": 1.6798,
+      "mean_token_accuracy": 0.602029812335968,
+      "step": 114000
+    },
+    {
+      "epoch": 0.7778691141803161,
+      "grad_norm": 1.1154159307479858,
+      "learning_rate": 0.00011174820138997398,
+      "loss": 1.7267,
+      "mean_token_accuracy": 0.5944099034667015,
+      "step": 114500
+    },
+    {
+      "epoch": 0.7812659225391821,
+      "grad_norm": 1.1828614473342896,
+      "learning_rate": 0.0001100565908272587,
+      "loss": 1.6915,
+      "mean_token_accuracy": 0.5983568229675293,
+      "step": 115000
+    },
+    {
+      "epoch": 0.7846627308980482,
+      "grad_norm": 1.8176969289779663,
+      "learning_rate": 0.0001083581866478257,
+      "loss": 1.7065,
+      "mean_token_accuracy": 0.595502304315567,
+      "step": 115500
+    },
+    {
+      "epoch": 0.7880595392569142,
+      "grad_norm": 1.3669997453689575,
+      "learning_rate": 0.00010666657608511044,
+      "loss": 1.6969,
+      "mean_token_accuracy": 0.5968110272884369,
+      "step": 116000
+    },
+    {
+      "epoch": 0.7914563476157802,
+      "grad_norm": 1.7632516622543335,
+      "learning_rate": 0.00010496817190567743,
+      "loss": 1.6838,
+      "mean_token_accuracy": 0.599950167298317,
+      "step": 116500
+    },
+    {
+      "epoch": 0.7948531559746462,
+      "grad_norm": 1.6624451875686646,
+      "learning_rate": 0.00010327656134296215,
+      "loss": 1.6832,
+      "mean_token_accuracy": 0.6004037878513336,
+      "step": 117000
+    },
+    {
+      "epoch": 0.7982499643335123,
+      "grad_norm": 1.3393323421478271,
+      "learning_rate": 0.00010158155397188802,
+      "loss": 1.6994,
+      "mean_token_accuracy": 0.5987599128484726,
+      "step": 117500
+    },
+    {
+      "epoch": 0.8016467726923783,
+      "grad_norm": 1.3584927320480347,
+      "learning_rate": 9.988314979245502e-05,
+      "loss": 1.7162,
+      "mean_token_accuracy": 0.592780797123909,
+      "step": 118000
+    },
+    {
+      "epoch": 0.8050435810512443,
+      "grad_norm": 1.4130176305770874,
+      "learning_rate": 9.8184745613022e-05,
+      "loss": 1.6721,
+      "mean_token_accuracy": 0.6051768786311149,
+      "step": 118500
+    },
+    {
+      "epoch": 0.8084403894101102,
+      "grad_norm": 1.2991234064102173,
+      "learning_rate": 9.6486341433589e-05,
+      "loss": 1.6824,
+      "mean_token_accuracy": 0.6021123319864273,
+      "step": 119000
+    },
+    {
+      "epoch": 0.8118371977689762,
+      "grad_norm": 1.8898921012878418,
+      "learning_rate": 9.479473087087372e-05,
+      "loss": 1.6978,
+      "mean_token_accuracy": 0.5959774954319,
+      "step": 119500
+    },
+    {
+      "epoch": 0.8152340061278422,
+      "grad_norm": 1.5865731239318848,
+      "learning_rate": 9.309632669144072e-05,
+      "loss": 1.6813,
+      "mean_token_accuracy": 0.5991669102907181,
+      "step": 120000
+    },
+    {
+      "epoch": 0.8186308144867083,
+      "grad_norm": 1.279549241065979,
+      "learning_rate": 9.140471612872545e-05,
+      "loss": 1.712,
+      "mean_token_accuracy": 0.5936280771493911,
+      "step": 120500
+    },
+    {
+      "epoch": 0.8220276228455743,
+      "grad_norm": 1.8203554153442383,
+      "learning_rate": 8.970631194929245e-05,
+      "loss": 1.708,
+      "mean_token_accuracy": 0.5971926473379136,
+      "step": 121000
+    },
+    {
+      "epoch": 0.8254244312044403,
+      "grad_norm": 1.6355427503585815,
+      "learning_rate": 8.801470138657717e-05,
+      "loss": 1.7075,
+      "mean_token_accuracy": 0.5944276967644692,
+      "step": 121500
+    },
+    {
+      "epoch": 0.8288212395633063,
+      "grad_norm": 1.8942941427230835,
+      "learning_rate": 8.631629720714417e-05,
+      "loss": 1.6955,
+      "mean_token_accuracy": 0.5952666994333268,
+      "step": 122000
+    },
+    {
+      "epoch": 0.8322180479221724,
+      "grad_norm": 2.0859923362731934,
+      "learning_rate": 8.462128983607004e-05,
+      "loss": 1.6754,
+      "mean_token_accuracy": 0.601703870356083,
+      "step": 122500
+    },
+    {
+      "epoch": 0.8356148562810384,
+      "grad_norm": 1.6707024574279785,
+      "learning_rate": 8.292288565663702e-05,
+      "loss": 1.6937,
+      "mean_token_accuracy": 0.5967540075778961,
+      "step": 123000
+    },
+    {
+      "epoch": 0.8390116646399044,
+      "grad_norm": 2.0465281009674072,
+      "learning_rate": 8.122448147720402e-05,
+      "loss": 1.6991,
+      "mean_token_accuracy": 0.5974170343875885,
+      "step": 123500
+    },
+    {
+      "epoch": 0.8424084729987703,
+      "grad_norm": 1.3448609113693237,
+      "learning_rate": 7.952607729777102e-05,
+      "loss": 1.6821,
+      "mean_token_accuracy": 0.5996044062376022,
+      "step": 124000
+    },
+    {
+      "epoch": 0.8458052813576363,
+      "grad_norm": 2.0044121742248535,
+      "learning_rate": 7.783446673505574e-05,
+      "loss": 1.6733,
+      "mean_token_accuracy": 0.6015715954899787,
+      "step": 124500
+    },
+    {
+      "epoch": 0.8492020897165024,
+      "grad_norm": 1.300785779953003,
+      "learning_rate": 7.613606255562273e-05,
+      "loss": 1.6738,
+      "mean_token_accuracy": 0.601664933681488,
+      "step": 125000
+    },
+    {
+      "epoch": 0.8525988980753684,
+      "grad_norm": 1.325310230255127,
+      "learning_rate": 7.444445199290747e-05,
+      "loss": 1.6814,
+      "mean_token_accuracy": 0.598460386633873,
+      "step": 125500
+    },
+    {
+      "epoch": 0.8559957064342344,
+      "grad_norm": 1.4887357950210571,
+      "learning_rate": 7.274604781347446e-05,
+      "loss": 1.6856,
+      "mean_token_accuracy": 0.5993021846413612,
+      "step": 126000
+    },
+    {
+      "epoch": 0.8593925147931004,
+      "grad_norm": 1.8545109033584595,
+      "learning_rate": 7.105443725075919e-05,
+      "loss": 1.6736,
+      "mean_token_accuracy": 0.6027230555415154,
+      "step": 126500
+    },
+    {
+      "epoch": 0.8627893231519664,
+      "grad_norm": 1.5627049207687378,
+      "learning_rate": 6.935603307132619e-05,
+      "loss": 1.6886,
+      "mean_token_accuracy": 0.5996944309473038,
+      "step": 127000
+    },
+    {
+      "epoch": 0.8661861315108325,
+      "grad_norm": 1.4192836284637451,
+      "learning_rate": 6.766442250861091e-05,
+      "loss": 1.6843,
+      "mean_token_accuracy": 0.5992154671549798,
+      "step": 127500
+    },
+    {
+      "epoch": 0.8695829398696985,
+      "grad_norm": 1.4279862642288208,
+      "learning_rate": 6.596601832917791e-05,
+      "loss": 1.6905,
+      "mean_token_accuracy": 0.5970825924873352,
+      "step": 128000
+    },
+    {
+      "epoch": 0.8729797482285644,
+      "grad_norm": 1.2963677644729614,
+      "learning_rate": 6.427101095810376e-05,
+      "loss": 1.6991,
+      "mean_token_accuracy": 0.5987445857524872,
+      "step": 128500
+    },
+    {
+      "epoch": 0.8763765565874304,
+      "grad_norm": 1.5592867136001587,
+      "learning_rate": 6.257260677867076e-05,
+      "loss": 1.694,
+      "mean_token_accuracy": 0.599335246026516,
+      "step": 129000
+    },
+    {
+      "epoch": 0.8797733649462964,
+      "grad_norm": 1.7764347791671753,
+      "learning_rate": 6.0877599407596624e-05,
+      "loss": 1.6948,
+      "mean_token_accuracy": 0.5971695448756218,
+      "step": 129500
+    },
+    {
+      "epoch": 0.8831701733051625,
+      "grad_norm": 2.150768756866455,
+      "learning_rate": 5.9179195228163615e-05,
+      "loss": 1.6751,
+      "mean_token_accuracy": 0.6033137931227684,
+      "step": 130000
+    },
+    {
+      "epoch": 0.8865669816640285,
+      "grad_norm": 1.6340703964233398,
+      "learning_rate": 5.7487584665448346e-05,
+      "loss": 1.6981,
+      "mean_token_accuracy": 0.5958724657893181,
+      "step": 130500
+    },
+    {
+      "epoch": 0.8899637900228945,
+      "grad_norm": 1.4169362783432007,
+      "learning_rate": 5.579257729437421e-05,
+      "loss": 1.6995,
+      "mean_token_accuracy": 0.597598332285881,
+      "step": 131000
+    },
+    {
+      "epoch": 0.8933605983817605,
+      "grad_norm": 2.037898540496826,
+      "learning_rate": 5.40941731149412e-05,
+      "loss": 1.6861,
+      "mean_token_accuracy": 0.5985950961112976,
+      "step": 131500
+    },
+    {
+      "epoch": 0.8967574067406265,
+      "grad_norm": 1.7336900234222412,
+      "learning_rate": 5.23957689355082e-05,
+      "loss": 1.6962,
+      "mean_token_accuracy": 0.5953020449876786,
+      "step": 132000
+    },
+    {
+      "epoch": 0.9001542150994926,
+      "grad_norm": 1.2341374158859253,
+      "learning_rate": 5.069736475607519e-05,
+      "loss": 1.6766,
+      "mean_token_accuracy": 0.5998600218892097,
+      "step": 132500
+    },
+    {
+      "epoch": 0.9035510234583585,
+      "grad_norm": 1.5542539358139038,
+      "learning_rate": 4.900575419335992e-05,
+      "loss": 1.6928,
+      "mean_token_accuracy": 0.5986370027065278,
+      "step": 133000
+    },
+    {
+      "epoch": 0.9069478318172245,
+      "grad_norm": 1.5925590991973877,
+      "learning_rate": 4.731074682228578e-05,
+      "loss": 1.6938,
+      "mean_token_accuracy": 0.5984804788827897,
+      "step": 133500
+    },
+    {
+      "epoch": 0.9103446401760905,
+      "grad_norm": 1.3265743255615234,
+      "learning_rate": 4.561234264285278e-05,
+      "loss": 1.6847,
+      "mean_token_accuracy": 0.5994028640985489,
+      "step": 134000
+    },
+    {
+      "epoch": 0.9137414485349565,
+      "grad_norm": 1.4869900941848755,
+      "learning_rate": 4.3913938463419766e-05,
+      "loss": 1.6823,
+      "mean_token_accuracy": 0.5991100046038628,
+      "step": 134500
+    },
+    {
+      "epoch": 0.9171382568938226,
+      "grad_norm": 1.5685834884643555,
+      "learning_rate": 4.2215534283986765e-05,
+      "loss": 1.7005,
+      "mean_token_accuracy": 0.5987723708748818,
+      "step": 135000
+    },
+    {
+      "epoch": 0.9205350652526886,
+      "grad_norm": 1.5705722570419312,
+      "learning_rate": 4.052392372127149e-05,
+      "loss": 1.6903,
+      "mean_token_accuracy": 0.5988448872566223,
+      "step": 135500
+    },
+    {
+      "epoch": 0.9239318736115546,
+      "grad_norm": 1.6409697532653809,
+      "learning_rate": 3.8828916350197357e-05,
+      "loss": 1.6797,
+      "mean_token_accuracy": 0.5988131073117257,
+      "step": 136000
+    },
+    {
+      "epoch": 0.9273286819704206,
+      "grad_norm": 1.5502301454544067,
+      "learning_rate": 3.7130512170764355e-05,
+      "loss": 1.683,
+      "mean_token_accuracy": 0.6030357847213745,
+      "step": 136500
+    },
+    {
+      "epoch": 0.9307254903292866,
+      "grad_norm": 1.7839597463607788,
+      "learning_rate": 3.543210799133135e-05,
+      "loss": 1.6953,
+      "mean_token_accuracy": 0.5989635826945305,
+      "step": 137000
+    },
+    {
+      "epoch": 0.9341222986881527,
+      "grad_norm": 1.411021113395691,
+      "learning_rate": 3.373370381189834e-05,
+      "loss": 1.6797,
+      "mean_token_accuracy": 0.5998646081089973,
+      "step": 137500
+    },
+    {
+      "epoch": 0.9375191070470186,
+      "grad_norm": 1.4591593742370605,
+      "learning_rate": 3.20386964408242e-05,
+      "loss": 1.6747,
+      "mean_token_accuracy": 0.6003830252885819,
+      "step": 138000
+    },
+    {
+      "epoch": 0.9409159154058846,
+      "grad_norm": 2.2815229892730713,
+      "learning_rate": 3.03402922613912e-05,
+      "loss": 1.6905,
+      "mean_token_accuracy": 0.599867588698864,
+      "step": 138500
+    },
+    {
+      "epoch": 0.9443127237647506,
+      "grad_norm": 1.820251703262329,
+      "learning_rate": 2.8648681698675926e-05,
+      "loss": 1.7074,
+      "mean_token_accuracy": 0.5943086860775948,
+      "step": 139000
+    },
+    {
+      "epoch": 0.9477095321236166,
+      "grad_norm": 1.7044726610183716,
+      "learning_rate": 2.695027751924292e-05,
+      "loss": 1.6714,
+      "mean_token_accuracy": 0.5998544086813926,
+      "step": 139500
+    },
+    {
+      "epoch": 0.9511063404824827,
+      "grad_norm": 1.4863064289093018,
+      "learning_rate": 2.525866695652765e-05,
+      "loss": 1.6815,
+      "mean_token_accuracy": 0.6012240989804268,
+      "step": 140000
+    },
+    {
+      "epoch": 0.9545031488413487,
+      "grad_norm": 1.409155011177063,
+      "learning_rate": 2.356365958545351e-05,
+      "loss": 1.6938,
+      "mean_token_accuracy": 0.5978354825377464,
+      "step": 140500
+    },
+    {
+      "epoch": 0.9578999572002147,
+      "grad_norm": 1.7919785976409912,
+      "learning_rate": 2.1865255406020502e-05,
+      "loss": 1.6816,
+      "mean_token_accuracy": 0.5987768666148185,
+      "step": 141000
+    },
+    {
+      "epoch": 0.9612967655590807,
+      "grad_norm": 1.4500936269760132,
+      "learning_rate": 2.0166851226587498e-05,
+      "loss": 1.6906,
+      "mean_token_accuracy": 0.5990597771406174,
+      "step": 141500
+    },
+    {
+      "epoch": 0.9646935739179467,
+      "grad_norm": 1.407329797744751,
+      "learning_rate": 1.8468447047154496e-05,
+      "loss": 1.6894,
+      "mean_token_accuracy": 0.5990225902795792,
+      "step": 142000
+    },
+    {
+      "epoch": 0.9680903822768127,
+      "grad_norm": 1.366939902305603,
+      "learning_rate": 1.677683648443922e-05,
+      "loss": 1.6899,
+      "mean_token_accuracy": 0.5980374491810798,
+      "step": 142500
+    },
+    {
+      "epoch": 0.9714871906356787,
+      "grad_norm": 1.512519121170044,
+      "learning_rate": 1.5078432305006216e-05,
+      "loss": 1.6739,
+      "mean_token_accuracy": 0.5996717846989632,
+      "step": 143000
+    },
+    {
+      "epoch": 0.9748839989945447,
+      "grad_norm": 1.878699541091919,
+      "learning_rate": 1.3386821742290945e-05,
+      "loss": 1.6916,
+      "mean_token_accuracy": 0.5977598049640656,
+      "step": 143500
+    },
+    {
+      "epoch": 0.9782808073534107,
+      "grad_norm": 1.404316782951355,
+      "learning_rate": 1.1688417562857938e-05,
+      "loss": 1.6802,
+      "mean_token_accuracy": 0.6025402237772941,
+      "step": 144000
+    },
+    {
+      "epoch": 0.9816776157122767,
+      "grad_norm": 1.619053840637207,
+      "learning_rate": 9.996807000142665e-06,
+      "loss": 1.6811,
+      "mean_token_accuracy": 0.599041444838047,
+      "step": 144500
+    },
+    {
+      "epoch": 0.9850744240711428,
+      "grad_norm": 1.4090548753738403,
+      "learning_rate": 8.298402820709663e-06,
+      "loss": 1.6758,
+      "mean_token_accuracy": 0.6014890729784965,
+      "step": 145000
+    },
+    {
+      "epoch": 0.9884712324300088,
+      "grad_norm": 1.9760881662368774,
+      "learning_rate": 6.606792257994388e-06,
+      "loss": 1.6872,
+      "mean_token_accuracy": 0.602891126692295,
+      "step": 145500
+    },
+    {
+      "epoch": 0.9918680407888748,
+      "grad_norm": 1.4356855154037476,
+      "learning_rate": 4.908388078561383e-06,
+      "loss": 1.6788,
+      "mean_token_accuracy": 0.5994058170318604,
+      "step": 146000
+    },
+    {
+      "epoch": 0.9952648491477408,
+      "grad_norm": 1.6398457288742065,
+      "learning_rate": 3.216777515846111e-06,
+      "loss": 1.6636,
+      "mean_token_accuracy": 0.6050380381941796,
+      "step": 146500
+    },
+    {
+      "epoch": 0.9986616575066067,
+      "grad_norm": 1.5855629444122314,
+      "learning_rate": 1.5183733364131064e-06,
+      "loss": 1.6748,
+      "mean_token_accuracy": 0.5996100574731826,
+      "step": 147000
+    },
+    {
+      "epoch": 1.0,
+      "mean_token_accuracy": 0.6006416203406861,
+      "step": 147197,
+      "total_flos": 9.396165357758054e+17,
+      "train_loss": 1.722812063241433,
+      "train_runtime": 12082.0087,
+      "train_samples_per_second": 24.366,
+      "train_steps_per_second": 12.183
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 147197,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 0,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 9.396165357758054e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37ad63a89bbec9e6e8ab733c69c2a514f924a6e691ab3520e1f5111e6b8726ab
+size 7032