End of training

README.md

@@ -0,0 +1,138 @@
+---
+license: mit
+base_model: microsoft/phi-2
+tags:
+- axolotl
+- generated_from_trainer
+model-index:
+- name: phi2-alpaca
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/OpenAccess-AI-Collective/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/OpenAccess-AI-Collective/axolotl)
+<details><summary>See axolotl config</summary>
+
+axolotl version: `0.3.0`
+```yaml
+base_model: microsoft/phi-2
+model_type: AutoModelForCausalLM
+tokenizer_type: AutoTokenizer
+trust_remote_code: true
+
+hub_model_id: openaccess-ai-collective/phi2-alpaca
+
+load_in_8bit: false
+load_in_4bit: false
+strict: false
+
+datasets:
+  - path: tatsu-lab/alpaca
+    type: alpaca
+
+dataset_prepared_path:
+val_set_size: 0.05
+output_dir: ./phi-sft-out
+
+sequence_len: 2048
+sample_packing: false  # currently unsupported
+pad_to_sequence_len:
+
+wandb_project: phi2
+wandb_entity: oaaic
+wandb_watch:
+wandb_name:
+wandb_log_model:
+
+gradient_accumulation_steps: 8
+micro_batch_size: 4
+num_epochs: 1
+optimizer: paged_adamw_8bit
+adam_beta2: 0.95
+adam_epsilon: 0.00001
+max_grad_norm: 1.0
+lr_scheduler: cosine
+learning_rate: 1e-5
+
+train_on_inputs: false
+group_by_length: false
+bf16: true
+fp16: false
+tf32: true
+
+gradient_checkpointing: true
+early_stopping_patience:
+resume_from_checkpoint:
+local_rank:
+logging_steps: 1
+xformers_attention:
+flash_attention: true
+
+warmup_steps: 100
+evals_per_epoch: 4
+saves_per_epoch: 1
+debug:
+deepspeed:
+weight_decay: 0.1
+fsdp:
+fsdp_config:
+resize_token_embeddings_to_32x: true
+special_tokens:
+  pad_token: "<|endoftext|>"
+
+```
+
+</details><br>
+
+# phi2-alpaca
+
+This model is a fine-tuned version of [microsoft/phi-2](https://huggingface.co/microsoft/phi-2) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.9343
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 1e-05
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.95) and epsilon=1e-05
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 1.3994        | 0.0   | 1    | 1.3199          |
+| 0.9532        | 0.25  | 386  | 0.9886          |
+| 0.8445        | 0.5   | 772  | 0.9421          |
+| 0.7303        | 0.75  | 1158 | 0.9343          |
+
+
+### Framework versions
+
+- Transformers 4.37.0.dev0
+- Pytorch 2.0.1+cu118
+- Datasets 2.16.1
+- Tokenizers 0.15.0

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:718c4d4fef30fd1d185b56541780fdadabb3b4161dee7979850df01671fc51d2
+size 4982468168

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0698eab71af465bac1e7529a28d5467052fc071ceb241397d367ba518d1024d4
+size 839190784

model.safetensors.index.json

@@ -0,0 +1,332 @@
+{
+  "metadata": {
+    "total_size": 5821624320
+  },
+  "weight_map": {
+    "lm_head.linear.bias": "model-00002-of-00002.safetensors",
+    "lm_head.linear.weight": "model-00002-of-00002.safetensors",
+    "lm_head.ln.bias": "model-00002-of-00002.safetensors",
+    "lm_head.ln.weight": "model-00002-of-00002.safetensors",
+    "transformer.embd.wte.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.0.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.0.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.0.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.0.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.0.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.1.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.1.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.1.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.1.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.1.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.10.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.10.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.10.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.10.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.10.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.11.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.11.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.11.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.11.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.11.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.12.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.12.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.12.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.12.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.12.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.13.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.13.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.13.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.13.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.13.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.14.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.14.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.14.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.14.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.14.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.14.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.14.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.14.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.14.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.14.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.15.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.15.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.15.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.15.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.15.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.15.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.15.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.15.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.15.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.15.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.16.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.16.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.16.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.16.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.16.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.16.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.16.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.16.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.16.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.16.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.17.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.17.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.17.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.17.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.17.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.17.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.17.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.17.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.17.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.17.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.18.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.18.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.18.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.18.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.18.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.18.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.18.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.18.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.18.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.18.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.19.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.19.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.19.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.19.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.19.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.19.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.19.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.19.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.19.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.19.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.2.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.2.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.2.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.2.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.2.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.20.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.20.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.20.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.20.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.20.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.20.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.20.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.20.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.20.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.20.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.21.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.21.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.21.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.21.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.21.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.21.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.21.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.21.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.21.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.21.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.22.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.22.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.22.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.22.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.22.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.22.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.22.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.22.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.22.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.22.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.23.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.23.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.23.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.23.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.23.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.23.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.23.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.23.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.23.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.23.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.24.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.24.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.24.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.24.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.24.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.24.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.24.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.24.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.24.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.24.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.25.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.25.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.25.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.25.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.25.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.25.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.25.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.25.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.25.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.25.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.26.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.26.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.26.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.26.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.26.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.26.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.26.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.26.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.26.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.26.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.27.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.27.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.27.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.27.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.27.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.27.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.27.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.27.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.27.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.27.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.28.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.28.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.28.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.28.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.28.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.28.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.28.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.28.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.28.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.28.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.29.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.29.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.29.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.29.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.29.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.29.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.29.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.29.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.29.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.29.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.3.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.3.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.3.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.3.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.3.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.30.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.30.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.30.mixer.Wqkv.bias": "model-00002-of-00002.safetensors",
+    "transformer.h.30.mixer.Wqkv.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.30.mixer.out_proj.bias": "model-00002-of-00002.safetensors",
+    "transformer.h.30.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.30.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "transformer.h.30.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.30.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "transformer.h.30.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.31.ln.bias": "model-00002-of-00002.safetensors",
+    "transformer.h.31.ln.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.31.mixer.Wqkv.bias": "model-00002-of-00002.safetensors",
+    "transformer.h.31.mixer.Wqkv.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.31.mixer.out_proj.bias": "model-00002-of-00002.safetensors",
+    "transformer.h.31.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.31.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "transformer.h.31.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.31.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "transformer.h.31.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.4.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.4.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.4.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.4.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.4.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.4.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.5.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.5.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.5.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.5.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.5.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.6.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.6.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.6.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.6.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.6.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.7.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.7.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.7.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.7.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.7.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.8.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.8.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.8.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.8.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.8.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.ln.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.9.ln.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.mixer.Wqkv.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.9.mixer.Wqkv.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.mixer.out_proj.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.9.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.9.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "transformer.h.9.mlp.fc2.weight": "model-00001-of-00002.safetensors"
+  }
+}

pytorch_model-00001-of-00002.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:88085fb4b2602024f93eb0806358293d851523e8fd7250b83c20e11c05bdac49
+oid sha256:9212946ae3c0f700b2b97c517a5268dafafbe3ff4ff9aa51eabe4dfc8ef32b06
 size 4982539832

pytorch_model-00002-of-00002.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:907200df5d2af505a3c03d62cdb52f7fbf6d7daec0087ec32d8e8b7c7a76b747
+oid sha256:a09dfb97537075b3e1a7d0b59af7c4a5ec480401eb91362d61e76aa1eab542e9
 size 839195995