Training in progress, step 100

adapter_config.json

@@ -8,18 +8,24 @@
   "init_lora_weights": true,
   "layers_pattern": null,
   "layers_to_transform": null,
+  "loftq_config": {},
   "lora_alpha": 16,
-  "lora_dropout": 0.1,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 64,
+  "r": 16,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "k_proj",
+    "v_proj",
     "o_proj",
-    "q_proj",
-    "v_proj"
+    "k_proj",
+    "up_proj",
+    "down_proj",
+    "gate_proj",
+    "q_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e78eb84b93d37fb31f9e5b3e0d20102164b6378daebfeb0aff2f6fa8634a4327
-size 218138576
+oid sha256:aef5642419366e4a7d1e2bc2aa55dc313138f0c45a38198b37a8d2211c23b261
+size 83946192

runs/Jan08_12-47-46_ip-26-0-162-180/events.out.tfevents.1704718276.ip-26-0-162-180.3383747.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91ec8ae962aaa3cabf30a3deb513886f950c4839491c572ec75694d6ac3bfe99
+size 8075

tokenizer_config.json

@@ -1,4 +1,6 @@
 {
+  "add_bos_token": true,
+  "add_eos_token": false,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
@@ -27,7 +29,6 @@
   },
   "additional_special_tokens": [],
   "bos_token": "<s>",
-  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
   "legacy": true,
@@ -37,5 +38,5 @@
   "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
-  "use_default_system_prompt": true
+  "use_default_system_prompt": false
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:72b89ac9ebc9cdd470ac419b06fb5dd5754f7768ecb3f2bb1bdbd79f65f6a3ef
-size 4664
+oid sha256:18c3c3786c425267646ab1a14fd749b26e46371a2ccdfaa5f251dfea781f3273
+size 4792