trainer_state.json

{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 92,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01,
      "grad_norm": 4.720525067180577,
      "learning_rate": 5e-08,
      "logits/chosen": -0.7466397881507874,
      "logits/rejected": -0.8448368310928345,
      "logps/chosen": -906.5225830078125,
      "logps/rejected": -1495.2481689453125,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.11,
      "grad_norm": 4.632852976113696,
      "learning_rate": 5e-07,
      "logits/chosen": -0.6482676863670349,
      "logits/rejected": -0.895437479019165,
      "logps/chosen": -785.9156494140625,
      "logps/rejected": -1450.85986328125,
      "loss": 0.6931,
      "rewards/accuracies": 0.4305555522441864,
      "rewards/chosen": -0.0001847905368776992,
      "rewards/margins": 0.0002896597725339234,
      "rewards/rejected": -0.0004744503239635378,
      "step": 10
    },
    {
      "epoch": 0.22,
      "grad_norm": 4.8356653345167455,
      "learning_rate": 4.818756127755237e-07,
      "logits/chosen": -0.621070384979248,
      "logits/rejected": -0.8771441578865051,
      "logps/chosen": -827.4130859375,
      "logps/rejected": -1481.9398193359375,
      "loss": 0.69,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.005515743978321552,
      "rewards/margins": 0.007809244096279144,
      "rewards/rejected": -0.002293500117957592,
      "step": 20
    },
    {
      "epoch": 0.33,
      "grad_norm": 4.694283849707894,
      "learning_rate": 4.301303984001967e-07,
      "logits/chosen": -0.6596123576164246,
      "logits/rejected": -0.8908526301383972,
      "logps/chosen": -898.34716796875,
      "logps/rejected": -1416.7698974609375,
      "loss": 0.6837,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.014291609637439251,
      "rewards/margins": 0.01851365529000759,
      "rewards/rejected": -0.0042220475152134895,
      "step": 30
    },
    {
      "epoch": 0.43,
      "grad_norm": 4.692609837251871,
      "learning_rate": 3.52267159292835e-07,
      "logits/chosen": -0.6181058287620544,
      "logits/rejected": -0.8527644276618958,
      "logps/chosen": -824.63037109375,
      "logps/rejected": -1475.505615234375,
      "loss": 0.6739,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0304950512945652,
      "rewards/margins": 0.044622790068387985,
      "rewards/rejected": -0.014127736911177635,
      "step": 40
    },
    {
      "epoch": 0.54,
      "grad_norm": 4.61676988011927,
      "learning_rate": 2.5957568342250883e-07,
      "logits/chosen": -0.6702030897140503,
      "logits/rejected": -0.8896721601486206,
      "logps/chosen": -850.77587890625,
      "logps/rejected": -1460.890869140625,
      "loss": 0.6652,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 0.04143432155251503,
      "rewards/margins": 0.061493802815675735,
      "rewards/rejected": -0.020059484988451004,
      "step": 50
    },
    {
      "epoch": 0.65,
      "grad_norm": 5.494690774889503,
      "learning_rate": 1.6549578039787434e-07,
      "logits/chosen": -0.6942325234413147,
      "logits/rejected": -0.8787949681282043,
      "logps/chosen": -877.8935546875,
      "logps/rejected": -1478.18505859375,
      "loss": 0.657,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.03511770814657211,
      "rewards/margins": 0.0625246912240982,
      "rewards/rejected": -0.027406994253396988,
      "step": 60
    },
    {
      "epoch": 0.76,
      "grad_norm": 6.124390659372961,
      "learning_rate": 8.366857495860869e-08,
      "logits/chosen": -0.6788981556892395,
      "logits/rejected": -0.888959527015686,
      "logps/chosen": -834.4332275390625,
      "logps/rejected": -1386.3309326171875,
      "loss": 0.6483,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 0.05471896007657051,
      "rewards/margins": 0.09840215742588043,
      "rewards/rejected": -0.04368319362401962,
      "step": 70
    },
    {
      "epoch": 0.87,
      "grad_norm": 4.871541625050212,
      "learning_rate": 2.5958610759736126e-08,
      "logits/chosen": -0.6581851243972778,
      "logits/rejected": -0.8908681869506836,
      "logps/chosen": -754.8712158203125,
      "logps/rejected": -1406.0399169921875,
      "loss": 0.6428,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.06393760442733765,
      "rewards/margins": 0.11334818601608276,
      "rewards/rejected": -0.04941058158874512,
      "step": 80
    },
    {
      "epoch": 0.98,
      "grad_norm": 5.867975591798305,
      "learning_rate": 7.335497040648897e-10,
      "logits/chosen": -0.7055361866950989,
      "logits/rejected": -0.8931668996810913,
      "logps/chosen": -883.2164916992188,
      "logps/rejected": -1499.0478515625,
      "loss": 0.6427,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.040136195719242096,
      "rewards/margins": 0.09972117841243744,
      "rewards/rejected": -0.05958498641848564,
      "step": 90
    },
    {
      "epoch": 1.0,
      "step": 92,
      "total_flos": 0.0,
      "train_loss": 0.6656996970591338,
      "train_runtime": 1292.3769,
      "train_samples_per_second": 4.552,
      "train_steps_per_second": 0.071
    }
  ],
  "logging_steps": 10,
  "max_steps": 92,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}