Convert the model to a SequenceClassification variant

README.md

@@ -2,7 +2,9 @@
 license: apache-2.0
 base_model:
 - Qwen/Qwen3-8B-Base
-library_name: transformers
+tags:
+- transformers
+- sentence-transformers
 pipeline_tag: text-ranking
 ---
 # Qwen3-Reranker-8B
@@ -11,6 +13,12 @@ pipeline_tag: text-ranking
     <img src="https://qianwen-res.oss-accelerate-overseas.aliyuncs.com/logo_qwen3.png" width="400"/>
 <p>
 
+> [!NOTE]
+> This is a copy of the [Qwen3-Reranker-8B](https://huggingface.co/Qwen/Qwen3-Reranker-8B) model, part of the [Qwen3 Reranker series](https://huggingface.co/collections/Qwen/qwen3-reranker-6841b22d0192d7ade9cdefea), modified as a sequence classification model instead. See [Updated Usage](#updated-usage) for details on how to use it, or [Original Usage](#original-usage) for the original usage.
+> 
+> See [this discussion](https://huggingface.co/Qwen/Qwen3-Reranker-0.6B/discussions/3) for details on the conversion approach.
+
+
 ## Highlights
 
 The Qwen3 Embedding model series is the latest proprietary model of the Qwen family, specifically designed for text embedding and ranking tasks. Building upon the dense foundational models of the Qwen3 series, it provides a comprehensive range of text embeddings and reranking models in various sizes (0.6B, 4B, and 8B). This series inherits the exceptional multilingual capabilities, long-text understanding, and reasoning skills of its foundational model. The Qwen3 Embedding series represents significant advancements in multiple text embedding and ranking tasks, including text retrieval, code retrieval, text classification, text clustering, and bitext mining.
@@ -57,7 +65,116 @@ With Transformers versions earlier than 4.51.0, you may encounter the following
 KeyError: 'qwen3'
 ```
 
-### Transformers Usage
+### Updated Usage
+
+#### Updated Sentence Transformers Usage
+
+```python
+# Requires transformers>=4.51.0
+from sentence_transformers import CrossEncoder
+
+
+def format_queries(query, instruction=None):
+    prefix = '<|im_start|>system\nJudge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be "yes" or "no".<|im_end|>\n<|im_start|>user\n'
+    if instruction is None:
+        instruction = (
+            "Given a web search query, retrieve relevant passages that answer the query"
+        )
+    return f"{prefix}<Instruct>: {instruction}\n<Query>: {query}\n"
+
+
+def format_document(document):
+    suffix = "<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
+    return f"<Document>: {document}{suffix}"
+
+
+model = CrossEncoder("tomaarsen/Qwen3-Reranker-8B-seq-cls")
+
+task = "Given a web search query, retrieve relevant passages that answer the query"
+
+queries = [
+    "Which planet is known as the Red Planet?",
+    "Which planet is known as the Red Planet?",
+    "Which planet is known as the Red Planet?",
+    "Which planet is known as the Red Planet?",
+]
+
+documents = [
+    "Venus is often called Earth's twin because of its similar size and proximity.",
+    "Mars, known for its reddish appearance, is often referred to as the Red Planet.",
+    "Jupiter, the largest planet in our solar system, has a prominent red spot.",
+    "Saturn, famous for its rings, is sometimes mistaken for the Red Planet.",
+]
+
+pairs = [
+    [format_queries(query, task), format_document(doc)]
+    for query, doc in zip(queries, documents)
+]
+scores = model.predict(pairs)
+print(scores.tolist())
+# [0.0003314583736937493, 0.9842268824577332, 0.004446804523468018, 0.009984465315937996]
+```
+
+#### Updated Transformers Usage
+
+```python
+# Requires transformers>=4.51.0
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+
+def format_instruction(instruction, query, doc):
+    prefix = '<|im_start|>system\nJudge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be "yes" or "no".<|im_end|>\n<|im_start|>user\n'
+    suffix = "<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
+    if instruction is None:
+        instruction = (
+            "Given a web search query, retrieve relevant passages that answer the query"
+        )
+    output = f"{prefix}<Instruct>: {instruction}\n<Query>: {query}\n<Document>: {doc}{suffix}"
+    return output
+
+
+tokenizer = AutoTokenizer.from_pretrained("tomaarsen/Qwen3-Reranker-8B-seq-cls", padding_side="left")
+model = AutoModelForSequenceClassification.from_pretrained("tomaarsen/Qwen3-Reranker-8B-seq-cls").eval()
+# We recommend enabling flash_attention_2 for better acceleration and memory saving.
+# model = AutoModelForSequenceClassification.from_pretrained("tomaarsen/Qwen3-Reranker-8B-seq-cls", torch_dtype=torch.float16, attn_implementation="flash_attention_2").cuda().eval()
+max_length = 8192
+
+task = "Given a web search query, retrieve relevant passages that answer the query"
+
+queries = [
+    "Which planet is known as the Red Planet?",
+    "Which planet is known as the Red Planet?",
+    "Which planet is known as the Red Planet?",
+    "Which planet is known as the Red Planet?",
+]
+
+documents = [
+    "Venus is often called Earth's twin because of its similar size and proximity.",
+    "Mars, known for its reddish appearance, is often referred to as the Red Planet.",
+    "Jupiter, the largest planet in our solar system, has a prominent red spot.",
+    "Saturn, famous for its rings, is sometimes mistaken for the Red Planet.",
+]
+
+pairs = [format_instruction(task, query, doc) for query, doc in zip(queries, documents)]
+inputs = tokenizer(
+    pairs,
+    padding=True,
+    truncation=True,
+    max_length=max_length,
+    return_tensors="pt",
+)
+logits = model(**inputs).logits.squeeze()
+print(logits.tolist())
+# [-8.011678695678711, 4.133551120758057, -5.411122798919678, -4.5966949462890625]
+
+scores = logits.sigmoid()
+print(scores.tolist())
+# [0.00033145773340947926, 0.9842268824577332, 0.004446760285645723, 0.009984418749809265]
+```
+
+### Original Usage
+
+#### Original Transformers Usage
 
 ```python
 # Requires transformers>=4.51.0
@@ -93,8 +210,8 @@ def compute_logits(inputs, **kwargs):
     return scores
 
 tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-Reranker-8B", padding_side='left')
-
 model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-Reranker-8B").eval()
+
 # We recommend enabling flash_attention_2 for better acceleration and memory saving.
 # model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-Reranker-8B", torch_dtype=torch.float16, attn_implementation="flash_attention_2").cuda().eval()
 
@@ -109,13 +226,18 @@ suffix_tokens = tokenizer.encode(suffix, add_special_tokens=False)
         
 task = 'Given a web search query, retrieve relevant passages that answer the query'
 
-queries = ["What is the capital of China?",
-    "Explain gravity",
+queries = [
+    "Which planet is known as the Red Planet?",
+    "Which planet is known as the Red Planet?",
+    "Which planet is known as the Red Planet?",
+    "Which planet is known as the Red Planet?",
 ]
 
 documents = [
-    "The capital of China is Beijing.",
-    "Gravity is a force that attracts two bodies towards each other. It gives weight to physical objects and is responsible for the movement of planets around the sun.",
+    "Venus is often called Earth's twin because of its similar size and proximity.",
+    "Mars, known for its reddish appearance, is often referred to as the Red Planet.",
+    "Jupiter, the largest planet in our solar system, has a prominent red spot.",
+    "Saturn, famous for its rings, is sometimes mistaken for the Red Planet.",
 ]
 
 pairs = [format_instruction(task, query, doc) for query, doc in zip(queries, documents)]
@@ -125,6 +247,7 @@ inputs = process_inputs(pairs)
 scores = compute_logits(inputs)
 
 print("scores: ", scores)
+# scores:  [0.00033198529854416847, 0.9842491745948792, 0.00445373822003603, 0.010004101321101189]
 ```
 
 📌 **Tip**: We recommend that developers customize the `instruct` according to their specific scenarios, tasks, and languages. Our tests have shown that in most retrieval scenarios, not using an `instruct` on the query side can lead to a drop in retrieval performance by approximately 1% to 5%.

config.json

@@ -1,6 +1,6 @@
 {
   "architectures": [
-    "Qwen3ForCausalLM"
+    "Qwen3ForSequenceClassification"
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
@@ -9,21 +9,28 @@
   "head_dim": 128,
   "hidden_act": "silu",
   "hidden_size": 4096,
+  "id2label": {
+    "0": "LABEL_0"
+  },
   "initializer_range": 0.02,
   "intermediate_size": 12288,
+  "label2id": {
+    "LABEL_0": 0
+  },
   "max_position_embeddings": 40960,
   "max_window_layers": 36,
   "model_type": "qwen3",
   "num_attention_heads": 32,
   "num_hidden_layers": 36,
   "num_key_value_heads": 8,
+  "pad_token_id": 151643,
   "rms_norm_eps": 1e-06,
   "rope_scaling": null,
   "rope_theta": 1000000,
   "sliding_window": null,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.51.3",
+  "transformers_version": "4.52.4",
   "use_cache": true,
   "use_sliding_window": false,
   "vocab_size": 151669

model-00001-of-00005.safetensors → model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:22cdfea4a13b7b3e866573800eeeb638fc38962940adf631d06dc03befed047a
-size 4027618768
+oid sha256:d1f41938fe22277ff3f1ccbc1fbca0cb336c02e89960f0d89cfc9ea4643a1dde
+size 4900070432

model-00002-of-00005.safetensors → model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d2163b74137e35b4614bd2aa5bf27bcb07de4ca61c6962495feb968385eb0df8
-size 4060268160
+oid sha256:c68f51633cc337d2fbddcb5ed02ff87d50bb3371e603f792adbc90931785cfae
+size 4915960368

model-00003-of-00005.safetensors → model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a5038caa78c817e8acce6806104869675938a33fd4e60ed038e9931d390d6989
-size 4043508680
+oid sha256:dd3540495bdcc8c06d7ac3da0576ff4413be93445067d4186fd0e04c8445d0c6
+size 4983068496

model-00004-of-00005.safetensors → model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:247f85538c5996d4c296291b0e4004f618c9b17ca8cdc25d1fc726567eb15803
-size 3003274088
+oid sha256:1b5d4206daf1844f494a1f4c57f535ff92760351cc7a0afb7d700038a9e7665b
+size 335578704

model-00005-of-00005.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8ba41b93c2e4ec8339ad16b000bc977fde196aeac054956cbfc8c0186ee6d4cf
-size 1242472576

model.safetensors.index.json

@@ -1,406 +1,406 @@
 {
   "metadata": {
-    "total_size": 16377096192
+    "total_size": 15134631936
   },
   "weight_map": {
-    "lm_head.weight": "model-00005-of-00005.safetensors",
-    "model.embed_tokens.weight": "model-00001-of-00005.safetensors",
-    "model.layers.0.input_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.1.input_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.10.input_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.11.input_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.12.input_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.13.input_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.14.input_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.15.input_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.16.input_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.17.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.18.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.18.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.18.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.19.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.19.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.19.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.2.input_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.20.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.20.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.20.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.21.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.21.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.21.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.22.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.22.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.22.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.23.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.24.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.25.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.26.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.27.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.28.input_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.28.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.28.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.28.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.28.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.29.input_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.29.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.29.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.29.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.29.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.29.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.29.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.29.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.29.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.29.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.29.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.3.input_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.30.input_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.30.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.30.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.30.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.30.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.30.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.30.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.30.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.30.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.30.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.30.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.31.input_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.31.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.31.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.31.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.31.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.31.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.31.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.31.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.31.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.31.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.32.input_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.32.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.32.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.32.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.32.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.32.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.32.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.32.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.32.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.32.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.33.input_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.33.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.33.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.33.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.33.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.34.input_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.34.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.34.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.34.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.34.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.35.input_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.35.self_attn.k_norm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.35.self_attn.q_norm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.4.input_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.5.input_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.6.input_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.7.input_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.8.input_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.8.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.8.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.9.input_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.9.self_attn.k_norm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.9.self_attn.q_norm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
-    "model.norm.weight": "model-00004-of-00005.safetensors"
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors",
+    "score.weight": "model-00004-of-00004.safetensors"
   }
 }