training_args.json

{
  "run_name": "kanana-cpt-qlora-sft-lora-concrete-250409",
  "model_name": "Onuii/DAMI-base-merge-0408",
  "dataset_name": "Onuii/DAMI-lecture-DB-Dataset-0408-concrete",
  "max_seq_length": 8192,
  "dtype": "bfloat16",
  "load_in_4bit": true,
  "r": 16,
  "lora_alpha": 32,
  "lora_dropout": 0.1,
  "use_rslora": true,
  "batch_size": 16,
  "gradient_accumulation_steps": 2,
  "learning_rate": 1e-05,
  "warmup_ratio": 0.05,
  "num_train_epochs": 4,
  "optimizer": "adamw_8bit",
  "weight_decay": 0.01,
  "lr_scheduler_type": "linear",
  "output_dir": "../outputs/kanana-cpt-qlora-sft-lora-concrete-250409",
  "logging_dir": "../logs",
  "logging_steps": 1,
  "save_steps": 200,
  "save_total_limit": 2,
  "report_to": "wandb",
  "use_gradient_checkpointing": true,
  "save_strategy": "steps",
  "random_seed": 562,
  "hyperparameter_dir": "../hyperparameters"
}