update train script

train_with_unsloth.py

@@ -28,6 +28,7 @@ import os
 import torch
 import random
 import logging
+import re
 
 logging.basicConfig(level=logging.WARNING)
 logger = logging.getLogger(__name__)
@@ -129,6 +130,18 @@ dataset = load_fitting_samples(
     seed=3407,
 )
 
+def clean_assistant_marker(example):
+    # collapse any "<|im_start|>assistant\n\n…\n\n" into "<|im_start|>assistant\n"
+    example["text"] = re.sub(
+        r"(<\|im_start\|>assistant)\n+",
+        r"\1\n",
+        example["text"]
+    )
+    return example
+
+# clean: <|im_start|>assistant\n\n -> <|im_start|>assistant\n
+dataset = dataset.map(clean_assistant_marker, batched=False)
+
 new_dataset = dataset.train_test_split(test_size=0.01)
 
 # Configure training arguments
@@ -139,7 +152,7 @@ training_args = SFTConfig(
     per_device_eval_batch_size=1,
     eval_accumulation_steps=4,
     evaluation_strategy="steps",
-    eval_steps=10,
+    eval_steps=100,
     save_strategy="steps",
     save_steps=500,
     load_best_model_at_end=True,